telecom-cdr-pipeline-ai/docker-compose.yml

x-runtime: &runtime
  API_HOST: ${API_HOST:-0.0.0.0}
  API_PORT: ${API_PORT:-8000}
  API_QUEUE_MAX_SIZE: ${API_QUEUE_MAX_SIZE:-10000}
  API_MAX_BODY_BYTES: ${API_MAX_BODY_BYTES:-65536}
  API_READ_TIMEOUT_SECONDS: ${API_READ_TIMEOUT_SECONDS:-10}
  SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-45}
  WORKER_STATS_HOST: ${WORKER_STATS_HOST:-0.0.0.0}
  WORKER_STATS_PORT: ${WORKER_STATS_PORT:-8001}
  WORKER_STATS_URL: ${WORKER_STATS_URL:-http://worker:${WORKER_STATS_PORT:-8001}/internal/stats}
  WORKER_INSTANCE: ${WORKER_INSTANCE:-worker-1}
  KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-redpanda:29092}
  KAFKA_RAW_TOPIC: ${KAFKA_RAW_TOPIC:-cdr.raw}
  KAFKA_DLQ_TOPIC: ${KAFKA_DLQ_TOPIC:-cdr.raw.dlq}
  KAFKA_CONSUMER_GROUP: ${KAFKA_CONSUMER_GROUP:-cdr-processing-group}
  KAFKA_AUTO_OFFSET_RESET: ${KAFKA_AUTO_OFFSET_RESET:-earliest}
  KAFKA_DELIVERY_TIMEOUT_SECONDS: ${KAFKA_DELIVERY_TIMEOUT_SECONDS:-10}
  CLICKHOUSE_HOST: ${CLICKHOUSE_HOST:-clickhouse}
  # CLICKHOUSE_PORT is the published host port; the server listens on 8123 internally.
  CLICKHOUSE_PORT: 8123
  CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
  CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-}
  CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-telecom}
  CLICKHOUSE_TABLE: ${CLICKHOUSE_TABLE:-cdr_records}
  BATCH_SIZE: ${BATCH_SIZE:-1000}
  BATCH_FLUSH_INTERVAL_SECONDS: ${BATCH_FLUSH_INTERVAL_SECONDS:-2}
  CH_RETRY_MAX_ATTEMPTS: ${CH_RETRY_MAX_ATTEMPTS:-3}
  CH_RETRY_INITIAL_DELAY_MS: ${CH_RETRY_INITIAL_DELAY_MS:-500}
  CH_RETRY_BACKOFF_FACTOR: ${CH_RETRY_BACKOFF_FACTOR:-2}
  LOG_LEVEL: ${LOG_LEVEL:-INFO}

services:
  redpanda:
    image: docker.redpanda.com/redpandadata/redpanda:v24.1.8
    command:
      - redpanda start
      - --smp 1
      - --memory 1G
      - --reserve-memory 0M
      - --overprovisioned
      - --node-id 0
      - --check=false
      - --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:${KAFKA_PORT:-9092}
      - --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://${KAFKA_ADVERTISED_HOST:-localhost}:${KAFKA_PORT:-9092}
    ports:
      - "${KAFKA_PORT:-9092}:${KAFKA_PORT:-9092}"
      - "${REDPANDA_ADMIN_PORT:-9644}:9644"
    volumes:
      - kafka-data:/var/lib/redpanda/data
    healthcheck:
      test: ["CMD", "rpk", "cluster", "health", "--exit-when-healthy"]
      interval: 5s
      timeout: 5s
      retries: 12

  clickhouse:
    image: clickhouse/clickhouse-server:24.3-alpine
    environment:
      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-}
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    ports:
      - "${CLICKHOUSE_PORT:-8123}:8123"
      - "${CLICKHOUSE_NATIVE_PORT:-9000}:9000"
    volumes:
      - clickhouse-data:/var/lib/clickhouse
    healthcheck:
      test: ["CMD-SHELL", "clickhouse-client --user \"$$CLICKHOUSE_USER\" --password \"$$CLICKHOUSE_PASSWORD\" --query 'SELECT 1'"]
      interval: 5s
      timeout: 5s
      retries: 12

  init:
    build:
      context: .
      dockerfile: Dockerfile.worker
    command: ["python", "-m", "app.common.bootstrap"]
    environment: *runtime
    depends_on:
      redpanda:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "python", "-c", "from app.common.bootstrap import SCHEMA; assert SCHEMA.is_file()"]
      interval: 5s
      timeout: 5s
      retries: 3
    restart: "no"

  api:
    build:
      context: .
      dockerfile: Dockerfile.api
    environment: *runtime
    ports:
      - "${API_PORT:-8000}:${API_PORT:-8000}"
    stop_grace_period: 60s
    depends_on:
      redpanda:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
      init:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['API_PORT'] + '/health')"]
      interval: 5s
      timeout: 5s
      retries: 12

  worker:
    build:
      context: .
      dockerfile: Dockerfile.worker
    environment: *runtime
    ports:
      - "${WORKER_STATS_PORT:-8001}:${WORKER_STATS_PORT:-8001}"
    stop_grace_period: 60s
    depends_on:
      redpanda:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
      init:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['WORKER_STATS_PORT'] + '/health')"]
      interval: 5s
      timeout: 5s
      retries: 12

volumes:
  kafka-data:
  clickhouse-data: