amiirkhl/telecom-cdr-pipeline-ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(docker): provision durable topics and canonical ClickHouse schema

Browse Source
This commit is contained in:
Amirhossein Khalili
2026-09-19 21:30:41 +03:30
parent 3b59d76f67
commit b5bc2d722f
7 changed files with 201 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
docker-compose.yml
View File

@@ -1,10 +1,37 @@
x-runtime: &runtime
API_HOST: ${API_HOST:-0.0.0.0}
API_PORT: ${API_PORT:-8000}
API_QUEUE_MAX_SIZE: ${API_QUEUE_MAX_SIZE:-10000}
API_MAX_BODY_BYTES: ${API_MAX_BODY_BYTES:-65536}
API_READ_TIMEOUT_SECONDS: ${API_READ_TIMEOUT_SECONDS:-10}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-45}
WORKER_STATS_HOST: ${WORKER_STATS_HOST:-0.0.0.0}
WORKER_STATS_PORT: ${WORKER_STATS_PORT:-8001}
WORKER_STATS_URL: ${WORKER_STATS_URL:-http://worker:${WORKER_STATS_PORT:-8001}/internal/stats}
WORKER_INSTANCE: ${WORKER_INSTANCE:-worker-1}
KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-redpanda:29092}
KAFKA_RAW_TOPIC: ${KAFKA_RAW_TOPIC:-cdr.raw}
KAFKA_DLQ_TOPIC: ${KAFKA_DLQ_TOPIC:-cdr.raw.dlq}
KAFKA_CONSUMER_GROUP: ${KAFKA_CONSUMER_GROUP:-cdr-processing-group}
KAFKA_AUTO_OFFSET_RESET: ${KAFKA_AUTO_OFFSET_RESET:-earliest}
KAFKA_DELIVERY_TIMEOUT_SECONDS: ${KAFKA_DELIVERY_TIMEOUT_SECONDS:-10}
CLICKHOUSE_HOST: ${CLICKHOUSE_HOST:-clickhouse}
# CLICKHOUSE_PORT is the published host port; the server listens on 8123 internally.
CLICKHOUSE_PORT: 8123
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-}
CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-telecom}
CLICKHOUSE_TABLE: ${CLICKHOUSE_TABLE:-cdr_records}
BATCH_SIZE: ${BATCH_SIZE:-1000}
BATCH_FLUSH_INTERVAL_SECONDS: ${BATCH_FLUSH_INTERVAL_SECONDS:-2}
CH_RETRY_MAX_ATTEMPTS: ${CH_RETRY_MAX_ATTEMPTS:-3}
CH_RETRY_INITIAL_DELAY_MS: ${CH_RETRY_INITIAL_DELAY_MS:-500}
CH_RETRY_BACKOFF_FACTOR: ${CH_RETRY_BACKOFF_FACTOR:-2}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
services:
# ==========================================
# Message Broker: Redpanda (Kafka-compatible)
# ==========================================
redpanda:
image: docker.redpanda.com/redpandadata/redpanda:v24.1.8
container_name: redpanda
command:
- redpanda start
- --smp 1
@@ -13,126 +40,83 @@ services:
- --overprovisioned
- --node-id 0
- --check=false
- --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
- --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
- --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:${KAFKA_PORT:-9092}
- --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://${KAFKA_ADVERTISED_HOST:-localhost}:${KAFKA_PORT:-9092}
ports:
- "9092:9092"
- "9644:9644"
- "${KAFKA_PORT:-9092}:${KAFKA_PORT:-9092}"
- "${REDPANDA_ADMIN_PORT:-9644}:9644"
volumes:
- kafka-data:/var/lib/redpanda/data
healthcheck:
test: ["CMD", "rpk", "cluster", "health"]
test: ["CMD", "rpk", "cluster", "health", "--exit-when-healthy"]
interval: 5s
timeout: 5s
retries: 12
start_period: 10s
# Automated topic provisioner ensuring exact partitions & compression
topic-init:
image: docker.redpanda.com/redpandadata/redpanda:v24.1.8
container_name: redpanda-topic-init
clickhouse:
image: clickhouse/clickhouse-server:24.3-alpine
environment:
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-}
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
ports:
- "${CLICKHOUSE_PORT:-8123}:8123"
- "${CLICKHOUSE_NATIVE_PORT:-9000}:9000"
volumes:
- clickhouse-data:/var/lib/clickhouse
healthcheck:
test: ["CMD-SHELL", "clickhouse-client --user \"$$CLICKHOUSE_USER\" --password \"$$CLICKHOUSE_PASSWORD\" --query 'SELECT 1'"]
interval: 5s
timeout: 5s
retries: 12
init:
build:
context: .
dockerfile: Dockerfile.worker
command: ["python", "-m", "app.common.bootstrap"]
environment: *runtime
depends_on:
redpanda:
condition: service_healthy
entrypoint: /bin/sh
command:
- -c
- |
rpk topic create cdr.raw -p 3 -r 1 -c compression.type=lz4 -c retention.ms=86400000 --brokers=redpanda:29092 || true
rpk topic create cdr.raw.dlq -p 1 -r 1 -c retention.ms=604800000 --brokers=redpanda:29092 || true
clickhouse:
condition: service_healthy
restart: "no"
# ==========================================
# Analytics Database: ClickHouse
# ==========================================
clickhouse:
image: clickhouse/clickhouse-server:24.3-alpine
container_name: clickhouse
ports:
- "8123:8123"
- "9000:9000"
environment:
CLICKHOUSE_DB: telecom
CLICKHOUSE_USER: default
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
volumes:
- ./docker/clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
healthcheck:
test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
interval: 5s
timeout: 5s
retries: 10
start_period: 10s
# ==========================================
# Service 1: Ingestion API Gate
# ==========================================
api:
build:
context: .
dockerfile: Dockerfile.api
container_name: ingestion-api
environment: *runtime
ports:
- "8000:8000"
environment:
API_HOST: 0.0.0.0
API_PORT: 8000
API_QUEUE_MAX_SIZE: 10000
WORKER_STATS_URL: http://worker:8001/internal/stats
KAFKA_BOOTSTRAP_SERVERS: redpanda:29092
KAFKA_RAW_TOPIC: cdr.raw
LOG_LEVEL: INFO
- "${API_PORT:-8000}:${API_PORT:-8000}"
stop_grace_period: 60s
depends_on:
redpanda:
condition: service_healthy
topic-init:
init:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"]
test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['API_PORT'] + '/health')"]
interval: 5s
timeout: 5s
retries: 10
start_period: 5s
retries: 12
# ==========================================
# Service 2: Processing Worker
# ==========================================
worker:
build:
context: .
dockerfile: Dockerfile.worker
container_name: processing-worker
environment: *runtime
ports:
- "8001:8001"
environment:
WORKER_INSTANCE: worker-1
WORKER_STATS_HOST: 0.0.0.0
WORKER_STATS_PORT: 8001
KAFKA_BOOTSTRAP_SERVERS: redpanda:29092
KAFKA_RAW_TOPIC: cdr.raw
KAFKA_DLQ_TOPIC: cdr.raw.dlq
KAFKA_CONSUMER_GROUP: cdr-processing-group
KAFKA_AUTO_OFFSET_RESET: earliest
CLICKHOUSE_HOST: clickhouse
CLICKHOUSE_PORT: 8123
CLICKHOUSE_USER: default
CLICKHOUSE_PASSWORD: ""
CLICKHOUSE_DATABASE: telecom
CLICKHOUSE_TABLE: cdr_records
BATCH_SIZE: 1000
BATCH_FLUSH_INTERVAL_SECONDS: 2
CH_RETRY_MAX_ATTEMPTS: 3
CH_RETRY_INITIAL_DELAY_MS: 500
CH_RETRY_BACKOFF_FACTOR: 2.0
LOG_LEVEL: INFO
- "${WORKER_STATS_PORT:-8001}:${WORKER_STATS_PORT:-8001}"
stop_grace_period: 60s
depends_on:
redpanda:
condition: service_healthy
topic-init:
init:
condition: service_completed_successfully
clickhouse:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8001/health')"]
test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['WORKER_STATS_PORT'] + '/health')"]
interval: 5s
timeout: 5s
retries: 10
start_period: 5s
retries: 12
volumes:
kafka-data:
clickhouse-data: