services: # ========================================== # Message Broker: Redpanda (Kafka-compatible) # ========================================== redpanda: image: docker.redpanda.com/redpandadata/redpanda:v24.1.8 container_name: redpanda command: - redpanda start - --smp 1 - --memory 1G - --reserve-memory 0M - --overprovisioned - --node-id 0 - --check=false - --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 - --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092 ports: - "9092:9092" - "9644:9644" healthcheck: test: ["CMD-SHELL", "rpk cluster health --brokers=localhost:9092 | grep -q 'Healthy:.*true' || exit 1"] interval: 5s timeout: 5s retries: 12 start_period: 10s # Automated topic provisioner ensuring exact partitions & compression topic-init: image: docker.redpanda.com/redpandadata/redpanda:v24.1.8 container_name: redpanda-topic-init depends_on: redpanda: condition: service_healthy entrypoint: /bin/sh command: - -c - | rpk topic create cdr.raw -p 3 -r 1 -c compression.type=lz4 -c retention.ms=86400000 --brokers=redpanda:29092 || true rpk topic create cdr.raw.dlq -p 1 -r 1 -c retention.ms=604800000 --brokers=redpanda:29092 || true restart: "no" # ========================================== # Analytics Database: ClickHouse # ========================================== clickhouse: image: clickhouse/clickhouse-server:24.3-alpine container_name: clickhouse ports: - "8123:8123" - "9000:9000" environment: CLICKHOUSE_DB: telecom CLICKHOUSE_USER: default CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 volumes: - ./docker/clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"] interval: 5s timeout: 5s retries: 10 start_period: 10s # ========================================== # Service 1: Ingestion API Gate # ========================================== api: build: context: . dockerfile: Dockerfile.api container_name: ingestion-api ports: - "8000:8000" environment: API_HOST: 0.0.0.0 API_PORT: 8000 API_QUEUE_MAX_SIZE: 10000 WORKER_STATS_URL: http://worker:8001/internal/stats KAFKA_BOOTSTRAP_SERVERS: redpanda:29092 KAFKA_RAW_TOPIC: cdr.raw LOG_LEVEL: INFO depends_on: redpanda: condition: service_healthy topic-init: condition: service_completed_successfully healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] interval: 5s timeout: 5s retries: 10 start_period: 5s # ========================================== # Service 2: Processing Worker # ========================================== worker: build: context: . dockerfile: Dockerfile.worker container_name: processing-worker ports: - "8001:8001" environment: WORKER_INSTANCE: worker-1 WORKER_STATS_HOST: 0.0.0.0 WORKER_STATS_PORT: 8001 KAFKA_BOOTSTRAP_SERVERS: redpanda:29092 KAFKA_RAW_TOPIC: cdr.raw KAFKA_DLQ_TOPIC: cdr.raw.dlq KAFKA_CONSUMER_GROUP: cdr-processing-group KAFKA_AUTO_OFFSET_RESET: earliest CLICKHOUSE_HOST: clickhouse CLICKHOUSE_PORT: 8123 CLICKHOUSE_USER: default CLICKHOUSE_PASSWORD: "" CLICKHOUSE_DATABASE: telecom CLICKHOUSE_TABLE: cdr_records BATCH_SIZE: 1000 BATCH_FLUSH_INTERVAL_SECONDS: 2 CH_RETRY_MAX_ATTEMPTS: 3 CH_RETRY_INITIAL_DELAY_MS: 500 CH_RETRY_BACKOFF_FACTOR: 2.0 LOG_LEVEL: INFO depends_on: redpanda: condition: service_healthy topic-init: condition: service_completed_successfully clickhouse: condition: service_healthy healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8001/health')"] interval: 5s timeout: 5s retries: 10 start_period: 5s