133 lines
3.7 KiB
YAML
133 lines
3.7 KiB
YAML
name: telecom-cdr-pipeline
|
|
|
|
services:
|
|
redpanda:
|
|
image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-latest}
|
|
|
|
container_name: telecom-redpanda
|
|
|
|
command:
|
|
- redpanda
|
|
- start
|
|
|
|
# Kafka listeners:
|
|
# 9092 -> other Docker containers
|
|
# 19092 -> applications running on our host machine
|
|
- --kafka-addr=internal://0.0.0.0:9092,external://0.0.0.0:19092
|
|
|
|
# These are the addresses Redpanda tells Kafka clients to use.
|
|
- --advertise-kafka-addr=internal://redpanda:9092,external://localhost:19092
|
|
|
|
# Internal Redpanda node communication.
|
|
- --rpc-addr=redpanda:33145
|
|
- --advertise-rpc-addr=redpanda:33145
|
|
|
|
# Suitable settings for local Docker development.
|
|
- --mode=dev-container
|
|
- --smp=1
|
|
- --default-log-level=info
|
|
|
|
ports:
|
|
- "19092:19092"
|
|
- "9644:9644" # Redpanda's Admin API
|
|
|
|
volumes:
|
|
- redpanda-data:/var/lib/redpanda/data
|
|
|
|
healthcheck:
|
|
test:
|
|
[
|
|
"CMD",
|
|
"rpk",
|
|
"cluster",
|
|
"health",
|
|
"--exit-when-healthy"
|
|
]
|
|
interval: 5s
|
|
timeout: 10s
|
|
retries: 10
|
|
start_period: 10s
|
|
|
|
redpanda-init:
|
|
image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-latest}
|
|
|
|
depends_on:
|
|
redpanda:
|
|
condition: service_healthy
|
|
|
|
environment:
|
|
KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_INTERNAL_BOOTSTRAP_SERVERS}
|
|
|
|
KAFKA_RAW_TOPIC: ${KAFKA_RAW_TOPIC}
|
|
KAFKA_RAW_TOPIC_PARTITIONS: ${KAFKA_RAW_TOPIC_PARTITIONS}
|
|
KAFKA_RAW_TOPIC_REPLICATION_FACTOR: ${KAFKA_RAW_TOPIC_REPLICATION_FACTOR}
|
|
KAFKA_RAW_TOPIC_RETENTION_MS: ${KAFKA_RAW_TOPIC_RETENTION_MS}
|
|
KAFKA_RAW_TOPIC_COMPRESSION: ${KAFKA_RAW_TOPIC_COMPRESSION}
|
|
|
|
KAFKA_DLQ_TOPIC: ${KAFKA_DLQ_TOPIC}
|
|
KAFKA_DLQ_TOPIC_PARTITIONS: ${KAFKA_DLQ_TOPIC_PARTITIONS}
|
|
KAFKA_DLQ_TOPIC_REPLICATION_FACTOR: ${KAFKA_DLQ_TOPIC_REPLICATION_FACTOR}
|
|
KAFKA_DLQ_TOPIC_RETENTION_MS: ${KAFKA_DLQ_TOPIC_RETENTION_MS}
|
|
|
|
entrypoint: ["/bin/bash", "-c"]
|
|
|
|
command:
|
|
- |
|
|
echo "Creating Kafka topics..."
|
|
|
|
rpk topic create "$$KAFKA_RAW_TOPIC" \
|
|
--if-not-exists \
|
|
--brokers "$$KAFKA_BOOTSTRAP_SERVERS" \
|
|
-p "$$KAFKA_RAW_TOPIC_PARTITIONS" \
|
|
-r "$$KAFKA_RAW_TOPIC_REPLICATION_FACTOR" \
|
|
-c retention.ms="$$KAFKA_RAW_TOPIC_RETENTION_MS" \
|
|
-c compression.type="$$KAFKA_RAW_TOPIC_COMPRESSION"
|
|
|
|
rpk topic create "$$KAFKA_DLQ_TOPIC" \
|
|
--if-not-exists \
|
|
--brokers "$$KAFKA_BOOTSTRAP_SERVERS" \
|
|
-p "$$KAFKA_DLQ_TOPIC_PARTITIONS" \
|
|
-r "$$KAFKA_DLQ_TOPIC_REPLICATION_FACTOR" \
|
|
-c retention.ms="$$KAFKA_DLQ_TOPIC_RETENTION_MS"
|
|
|
|
echo "Kafka topics initialized successfully."
|
|
|
|
clickhouse:
|
|
image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-latest}
|
|
|
|
container_name: telecom-clickhouse
|
|
|
|
environment:
|
|
CLICKHOUSE_DB: ${CLICKHOUSE_DATABASE}
|
|
CLICKHOUSE_USER: ${CLICKHOUSE_USER}
|
|
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
|
|
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
|
CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS: 1
|
|
|
|
ports:
|
|
- "${CLICKHOUSE_HTTP_HOST_PORT:-8123}:8123"
|
|
|
|
volumes:
|
|
- clickhouse-data:/var/lib/clickhouse
|
|
- ./infrastructure/clickhouse/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
|
|
|
|
ulimits:
|
|
nofile:
|
|
soft: 262144
|
|
hard: 262144
|
|
|
|
healthcheck:
|
|
test:
|
|
[
|
|
"CMD-SHELL",
|
|
"clickhouse-client --user \"$$CLICKHOUSE_USER\" --password \"$$CLICKHOUSE_PASSWORD\" --query \"SELECT 1\""
|
|
]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 10
|
|
start_period: 10s
|
|
|
|
volumes:
|
|
redpanda-data:
|
|
clickhouse-data:
|