From 17c6c1e427be0cc880b82bed992257dc84070c70 Mon Sep 17 00:00:00 2001 From: Amirhossein Khalili Date: Sat, 19 Sep 2026 13:27:55 +0330 Subject: [PATCH] feat(docker): complete Phase 4 - Dockerfiles and docker-compose orchestration with healthchecks --- Dockerfile.api | 15 ++ Dockerfile.worker | 15 ++ docker-compose.yml | 138 ++++++++++++++++++ .../changes/telecom-cdr-pipeline/tasks.md | 4 +- 4 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.api create mode 100644 Dockerfile.worker create mode 100644 docker-compose.yml diff --git a/Dockerfile.api b/Dockerfile.api new file mode 100644 index 0000000..d2d33cd --- /dev/null +++ b/Dockerfile.api @@ -0,0 +1,15 @@ +FROM python:3.12-slim + +WORKDIR /app + +ENV PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ /app/app/ + +EXPOSE 8000 + +CMD ["python", "-m", "app.api.server"] diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 0000000..465120b --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,15 @@ +FROM python:3.12-slim + +WORKDIR /app + +ENV PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ /app/app/ + +EXPOSE 8001 + +CMD ["python", "-m", "app.worker.consumer"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2c404b7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,138 @@ +services: + # ========================================== + # Message Broker: Redpanda (Kafka-compatible) + # ========================================== + redpanda: + image: docker.redpanda.com/redpandadata/redpanda:v24.1.8 + container_name: redpanda + command: + - redpanda start + - --smp 1 + - --memory 1G + - --reserve-memory 0M + - --overprovisioned + - --node-id 0 + - --check=false + - --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092 + ports: + - "9092:9092" + - "9644:9644" + healthcheck: + test: ["CMD-SHELL", "rpk cluster health --brokers=localhost:9092 | grep -q 'Healthy:.*true' || exit 1"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 10s + + # Automated topic provisioner ensuring exact partitions & compression + topic-init: + image: docker.redpanda.com/redpandadata/redpanda:v24.1.8 + container_name: redpanda-topic-init + depends_on: + redpanda: + condition: service_healthy + entrypoint: /bin/sh + command: + - -c + - | + rpk topic create cdr.raw -p 3 -r 1 -c compression.type=lz4 -c retention.ms=86400000 --brokers=redpanda:29092 || true + rpk topic create cdr.raw.dlq -p 1 -r 1 -c retention.ms=604800000 --brokers=redpanda:29092 || true + restart: "no" + + # ========================================== + # Analytics Database: ClickHouse + # ========================================== + clickhouse: + image: clickhouse/clickhouse-server:24.3-alpine + container_name: clickhouse + ports: + - "8123:8123" + - "9000:9000" + environment: + CLICKHOUSE_DB: telecom + CLICKHOUSE_USER: default + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + volumes: + - ./docker/clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + + # ========================================== + # Service 1: Ingestion API Gate + # ========================================== + api: + build: + context: . + dockerfile: Dockerfile.api + container_name: ingestion-api + ports: + - "8000:8000" + environment: + API_HOST: 0.0.0.0 + API_PORT: 8000 + API_QUEUE_MAX_SIZE: 10000 + WORKER_STATS_URL: http://worker:8001/internal/stats + KAFKA_BOOTSTRAP_SERVERS: redpanda:29092 + KAFKA_RAW_TOPIC: cdr.raw + LOG_LEVEL: INFO + depends_on: + redpanda: + condition: service_healthy + topic-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 5s + + # ========================================== + # Service 2: Processing Worker + # ========================================== + worker: + build: + context: . + dockerfile: Dockerfile.worker + container_name: processing-worker + ports: + - "8001:8001" + environment: + WORKER_INSTANCE: worker-1 + WORKER_STATS_HOST: 0.0.0.0 + WORKER_STATS_PORT: 8001 + KAFKA_BOOTSTRAP_SERVERS: redpanda:29092 + KAFKA_RAW_TOPIC: cdr.raw + KAFKA_DLQ_TOPIC: cdr.raw.dlq + KAFKA_CONSUMER_GROUP: cdr-processing-group + KAFKA_AUTO_OFFSET_RESET: earliest + CLICKHOUSE_HOST: clickhouse + CLICKHOUSE_PORT: 8123 + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + CLICKHOUSE_DATABASE: telecom + CLICKHOUSE_TABLE: cdr_records + BATCH_SIZE: 1000 + BATCH_FLUSH_INTERVAL_SECONDS: 2 + CH_RETRY_MAX_ATTEMPTS: 3 + CH_RETRY_INITIAL_DELAY_MS: 500 + CH_RETRY_BACKOFF_FACTOR: 2.0 + LOG_LEVEL: INFO + depends_on: + redpanda: + condition: service_healthy + topic-init: + condition: service_completed_successfully + clickhouse: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8001/health')"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 5s diff --git a/openspec/changes/telecom-cdr-pipeline/tasks.md b/openspec/changes/telecom-cdr-pipeline/tasks.md index 1c6bcb0..8b91d99 100644 --- a/openspec/changes/telecom-cdr-pipeline/tasks.md +++ b/openspec/changes/telecom-cdr-pipeline/tasks.md @@ -24,8 +24,8 @@ ## 4. Containerization & Orchestration -- [ ] 4.1 Create lightweight Dockerfiles for Ingestion API and Processing Worker without framework dependencies. -- [ ] 4.2 Create `docker-compose.yml` defining Kafka/Redpanda, ClickHouse, Ingestion API, and Processing Worker with healthcheck blocks and `depends_on: { condition: service_healthy }`. +- [x] 4.1 Create lightweight Dockerfiles for Ingestion API and Processing Worker without framework dependencies. +- [x] 4.2 Create `docker-compose.yml` defining Kafka/Redpanda, ClickHouse, Ingestion API, and Processing Worker with healthcheck blocks and `depends_on: { condition: service_healthy }`. ## 5. Verification & Test Traffic Generator