feat(docker): complete Phase 4 - Dockerfiles and docker-compose orchestration with healthchecks

Dockerfile.api

@@ -0,0 +1,15 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY app/ /app/app/
+
+EXPOSE 8000
+
+CMD ["python", "-m", "app.api.server"]

Dockerfile.worker

@@ -0,0 +1,15 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY app/ /app/app/
+
+EXPOSE 8001
+
+CMD ["python", "-m", "app.worker.consumer"]

docker-compose.yml

@@ -0,0 +1,138 @@
+services:
+  # ==========================================
+  # Message Broker: Redpanda (Kafka-compatible)
+  # ==========================================
+  redpanda:
+    image: docker.redpanda.com/redpandadata/redpanda:v24.1.8
+    container_name: redpanda
+    command:
+      - redpanda start
+      - --smp 1
+      - --memory 1G
+      - --reserve-memory 0M
+      - --overprovisioned
+      - --node-id 0
+      - --check=false
+      - --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
+      - --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
+    ports:
+      - "9092:9092"
+      - "9644:9644"
+    healthcheck:
+      test: ["CMD-SHELL", "rpk cluster health --brokers=localhost:9092 | grep -q 'Healthy:.*true' || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+      start_period: 10s
+
+  # Automated topic provisioner ensuring exact partitions & compression
+  topic-init:
+    image: docker.redpanda.com/redpandadata/redpanda:v24.1.8
+    container_name: redpanda-topic-init
+    depends_on:
+      redpanda:
+        condition: service_healthy
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - |
+        rpk topic create cdr.raw -p 3 -r 1 -c compression.type=lz4 -c retention.ms=86400000 --brokers=redpanda:29092 || true
+        rpk topic create cdr.raw.dlq -p 1 -r 1 -c retention.ms=604800000 --brokers=redpanda:29092 || true
+    restart: "no"
+
+  # ==========================================
+  # Analytics Database: ClickHouse
+  # ==========================================
+  clickhouse:
+    image: clickhouse/clickhouse-server:24.3-alpine
+    container_name: clickhouse
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    environment:
+      CLICKHOUSE_DB: telecom
+      CLICKHOUSE_USER: default
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    volumes:
+      - ./docker/clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
+  # ==========================================
+  # Service 1: Ingestion API Gate
+  # ==========================================
+  api:
+    build:
+      context: .
+      dockerfile: Dockerfile.api
+    container_name: ingestion-api
+    ports:
+      - "8000:8000"
+    environment:
+      API_HOST: 0.0.0.0
+      API_PORT: 8000
+      API_QUEUE_MAX_SIZE: 10000
+      WORKER_STATS_URL: http://worker:8001/internal/stats
+      KAFKA_BOOTSTRAP_SERVERS: redpanda:29092
+      KAFKA_RAW_TOPIC: cdr.raw
+      LOG_LEVEL: INFO
+    depends_on:
+      redpanda:
+        condition: service_healthy
+      topic-init:
+        condition: service_completed_successfully
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 5s
+
+  # ==========================================
+  # Service 2: Processing Worker
+  # ==========================================
+  worker:
+    build:
+      context: .
+      dockerfile: Dockerfile.worker
+    container_name: processing-worker
+    ports:
+      - "8001:8001"
+    environment:
+      WORKER_INSTANCE: worker-1
+      WORKER_STATS_HOST: 0.0.0.0
+      WORKER_STATS_PORT: 8001
+      KAFKA_BOOTSTRAP_SERVERS: redpanda:29092
+      KAFKA_RAW_TOPIC: cdr.raw
+      KAFKA_DLQ_TOPIC: cdr.raw.dlq
+      KAFKA_CONSUMER_GROUP: cdr-processing-group
+      KAFKA_AUTO_OFFSET_RESET: earliest
+      CLICKHOUSE_HOST: clickhouse
+      CLICKHOUSE_PORT: 8123
+      CLICKHOUSE_USER: default
+      CLICKHOUSE_PASSWORD: ""
+      CLICKHOUSE_DATABASE: telecom
+      CLICKHOUSE_TABLE: cdr_records
+      BATCH_SIZE: 1000
+      BATCH_FLUSH_INTERVAL_SECONDS: 2
+      CH_RETRY_MAX_ATTEMPTS: 3
+      CH_RETRY_INITIAL_DELAY_MS: 500
+      CH_RETRY_BACKOFF_FACTOR: 2.0
+      LOG_LEVEL: INFO
+    depends_on:
+      redpanda:
+        condition: service_healthy
+      topic-init:
+        condition: service_completed_successfully
+      clickhouse:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8001/health')"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 5s

openspec/changes/telecom-cdr-pipeline/tasks.md

@@ -24,8 +24,8 @@
 
 ## 4. Containerization & Orchestration
 
-- [ ] 4.1 Create lightweight Dockerfiles for Ingestion API and Processing Worker without framework dependencies.
+- [x] 4.1 Create lightweight Dockerfiles for Ingestion API and Processing Worker without framework dependencies.
-- [ ] 4.2 Create `docker-compose.yml` defining Kafka/Redpanda, ClickHouse, Ingestion API, and Processing Worker with healthcheck blocks and `depends_on: { condition: service_healthy }`.
+- [x] 4.2 Create `docker-compose.yml` defining Kafka/Redpanda, ClickHouse, Ingestion API, and Processing Worker with healthcheck blocks and `depends_on: { condition: service_healthy }`.
 
 ## 5. Verification & Test Traffic Generator