telecom-cdr-pipeline/docker-compose.yml

name: telecom-cdr-pipeline

services:
  redpanda:
    image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-v26.2.3}
    container_name: telecom-redpanda

    command:
      - redpanda
      - start
      - --kafka-addr=internal://0.0.0.0:9092,external://0.0.0.0:19092
      - --advertise-kafka-addr=internal://redpanda:9092,external://localhost:19092
      - --rpc-addr=redpanda:33145
      - --advertise-rpc-addr=redpanda:33145
      - --mode=dev-container
      - --smp=1
      - --default-log-level=info

    ports:
      - "19092:19092"
      - "9644:9644"

    volumes:
      - redpanda-data:/var/lib/redpanda/data

    healthcheck:
      test:
        [
          "CMD",
          "rpk",
          "cluster",
          "health",
          "--exit-when-healthy"
        ]
      interval: 5s
      timeout: 10s
      retries: 10
      start_period: 10s

    restart: unless-stopped


  redpanda-init:
    image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-v26.2.3}
    container_name: telecom-redpanda-init

    depends_on:
      redpanda:
        condition: service_healthy

    environment:
      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_INTERNAL_BOOTSTRAP_SERVERS:-redpanda:9092}

      KAFKA_RAW_TOPIC: ${KAFKA_RAW_TOPIC:-cdr.raw}
      KAFKA_RAW_TOPIC_PARTITIONS: ${KAFKA_RAW_TOPIC_PARTITIONS:-3}
      KAFKA_RAW_TOPIC_REPLICATION_FACTOR: ${KAFKA_RAW_TOPIC_REPLICATION_FACTOR:-1}
      KAFKA_RAW_TOPIC_RETENTION_MS: ${KAFKA_RAW_TOPIC_RETENTION_MS:-86400000}
      KAFKA_RAW_TOPIC_COMPRESSION: ${KAFKA_RAW_TOPIC_COMPRESSION:-lz4}

      KAFKA_DLQ_TOPIC: ${KAFKA_DLQ_TOPIC:-cdr.raw.dlq}
      KAFKA_DLQ_TOPIC_PARTITIONS: ${KAFKA_DLQ_TOPIC_PARTITIONS:-1}
      KAFKA_DLQ_TOPIC_REPLICATION_FACTOR: ${KAFKA_DLQ_TOPIC_REPLICATION_FACTOR:-1}
      KAFKA_DLQ_TOPIC_RETENTION_MS: ${KAFKA_DLQ_TOPIC_RETENTION_MS:-604800000}

    entrypoint:
      - /bin/bash
      - -c

    command:
      - |
        echo "Creating Kafka topics..."

        rpk topic create "$$KAFKA_RAW_TOPIC" \
          --if-not-exists \
          --brokers "$$KAFKA_BOOTSTRAP_SERVERS" \
          -p "$$KAFKA_RAW_TOPIC_PARTITIONS" \
          -r "$$KAFKA_RAW_TOPIC_REPLICATION_FACTOR" \
          -c retention.ms="$$KAFKA_RAW_TOPIC_RETENTION_MS" \
          -c compression.type="$$KAFKA_RAW_TOPIC_COMPRESSION"

        rpk topic create "$$KAFKA_DLQ_TOPIC" \
          --if-not-exists \
          --brokers "$$KAFKA_BOOTSTRAP_SERVERS" \
          -p "$$KAFKA_DLQ_TOPIC_PARTITIONS" \
          -r "$$KAFKA_DLQ_TOPIC_REPLICATION_FACTOR" \
          -c retention.ms="$$KAFKA_DLQ_TOPIC_RETENTION_MS"

        echo "Kafka topics initialized successfully."

    restart: "no"


  clickhouse:
    image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-26.8.7.19}
    container_name: telecom-clickhouse

    environment:
      CLICKHOUSE_DB: ${CLICKHOUSE_DATABASE:-telecom}
      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-telecom}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-telecom_dev_password}

      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
      CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS: 1

    ports:
      - "${CLICKHOUSE_HTTP_HOST_PORT:-8123}:8123"

    volumes:
      - clickhouse-data:/var/lib/clickhouse
      - ./infrastructure/clickhouse/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro

    ulimits:
      nofile:
        soft: 262144
        hard: 262144

    healthcheck:
      test:
        [
          "CMD-SHELL",
          "clickhouse-client --user \"$$CLICKHOUSE_USER\" --password \"$$CLICKHOUSE_PASSWORD\" --query \"SELECT 1\""
        ]
      interval: 5s
      timeout: 5s
      retries: 10
      start_period: 10s

    restart: unless-stopped


  api:
    build:
      context: .
      dockerfile: Dockerfile.api

    container_name: telecom-api

    depends_on:
      redpanda:
        condition: service_healthy

      redpanda-init:
        condition: service_completed_successfully

    environment:
      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_INTERNAL_BOOTSTRAP_SERVERS:-redpanda:9092}
      KAFKA_RAW_TOPIC: ${KAFKA_RAW_TOPIC:-cdr.raw}

      API_QUEUE_SIZE: ${API_QUEUE_SIZE:-10000}

      KAFKA_DELIVERY_TIMEOUT_SECONDS: ${KAFKA_DELIVERY_TIMEOUT_SECONDS:-4}
      KAFKA_MESSAGE_TIMEOUT_MS: ${KAFKA_MESSAGE_TIMEOUT_MS:-3000}

      KAFKA_LINGER_MS: ${KAFKA_LINGER_MS:-5}
      KAFKA_BATCH_SIZE: ${KAFKA_BATCH_SIZE:-131072}
      KAFKA_PRODUCER_MAX_MESSAGES: ${KAFKA_PRODUCER_MAX_MESSAGES:-100000}

      WORKER_STATS_URL: http://worker:8001/internal/stats
      WORKER_STATS_TIMEOUT_SECONDS: ${WORKER_STATS_TIMEOUT_SECONDS:-2}

    ports:
      - "${API_HOST_PORT:-8000}:8000"

    healthcheck:
      test:
        [
          "CMD",
          "python",
          "-c",
          "import urllib.request; urllib.request.urlopen('http://localhost:8000/health', timeout=2)"
        ]
      interval: 5s
      timeout: 3s
      retries: 10
      start_period: 5s

    restart: unless-stopped


  worker:
    build:
      context: .
      dockerfile: Dockerfile.worker

    container_name: telecom-worker

    depends_on:
      redpanda:
        condition: service_healthy

      redpanda-init:
        condition: service_completed_successfully

      clickhouse:
        condition: service_healthy

    environment:
      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_INTERNAL_BOOTSTRAP_SERVERS:-redpanda:9092}

      KAFKA_RAW_TOPIC: ${KAFKA_RAW_TOPIC:-cdr.raw}
      KAFKA_DLQ_TOPIC: ${KAFKA_DLQ_TOPIC:-cdr.raw.dlq}

      KAFKA_CONSUMER_GROUP: ${KAFKA_CONSUMER_GROUP:-cdr-processing-workers}
      KAFKA_AUTO_OFFSET_RESET: ${KAFKA_AUTO_OFFSET_RESET:-earliest}
      KAFKA_CONSUMER_POLL_TIMEOUT_SECONDS: ${KAFKA_CONSUMER_POLL_TIMEOUT_SECONDS:-1}

      KAFKA_DLQ_DELIVERY_TIMEOUT_SECONDS: ${KAFKA_DLQ_DELIVERY_TIMEOUT_SECONDS:-5}
      KAFKA_MESSAGE_TIMEOUT_MS: ${KAFKA_MESSAGE_TIMEOUT_MS:-3000}

      WORKER_INSTANCE_ID: ${WORKER_INSTANCE_ID:-worker-1}

      WORKER_HTTP_HOST: 0.0.0.0
      WORKER_HTTP_PORT: 8001

      CLICKHOUSE_HOST: ${CLICKHOUSE_INTERNAL_HOST:-clickhouse}
      CLICKHOUSE_PORT: ${CLICKHOUSE_PORT:-8123}

      CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-telecom}
      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-telecom}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-telecom_dev_password}

      BATCH_SIZE: ${BATCH_SIZE:-1000}
      BATCH_FLUSH_INTERVAL_SECONDS: ${BATCH_FLUSH_INTERVAL_SECONDS:-2}

      CLICKHOUSE_MAX_INSERT_ATTEMPTS: ${CLICKHOUSE_MAX_INSERT_ATTEMPTS:-3}
      CLICKHOUSE_RETRY_BASE_DELAY_SECONDS: ${CLICKHOUSE_RETRY_BASE_DELAY_SECONDS:-0.5}

      CLICKHOUSE_CONNECT_TIMEOUT_SECONDS: ${CLICKHOUSE_CONNECT_TIMEOUT_SECONDS:-5}
      CLICKHOUSE_SEND_RECEIVE_TIMEOUT_SECONDS: ${CLICKHOUSE_SEND_RECEIVE_TIMEOUT_SECONDS:-10}

    ports:
      - "${WORKER_HTTP_HOST_PORT:-8001}:8001"

    healthcheck:
      test:
        [
          "CMD",
          "python",
          "-c",
          "import urllib.request; urllib.request.urlopen('http://localhost:8001/health', timeout=2)"
        ]
      interval: 5s
      timeout: 3s
      retries: 10
      start_period: 5s

    restart: unless-stopped


volumes:
  redpanda-data:
  clickhouse-data: