139 lines
4.1 KiB
YAML
139 lines
4.1 KiB
YAML
services:
|
|
# ==========================================
|
|
# Message Broker: Redpanda (Kafka-compatible)
|
|
# ==========================================
|
|
redpanda:
|
|
image: docker.redpanda.com/redpandadata/redpanda:v24.1.8
|
|
container_name: redpanda
|
|
command:
|
|
- redpanda start
|
|
- --smp 1
|
|
- --memory 1G
|
|
- --reserve-memory 0M
|
|
- --overprovisioned
|
|
- --node-id 0
|
|
- --check=false
|
|
- --kafka-addr PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
|
|
- --advertise-kafka-addr PLAINTEXT://redpanda:29092,OUTSIDE://localhost:9092
|
|
ports:
|
|
- "9092:9092"
|
|
- "9644:9644"
|
|
healthcheck:
|
|
test: ["CMD", "rpk", "cluster", "health"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 12
|
|
start_period: 10s
|
|
|
|
# Automated topic provisioner ensuring exact partitions & compression
|
|
topic-init:
|
|
image: docker.redpanda.com/redpandadata/redpanda:v24.1.8
|
|
container_name: redpanda-topic-init
|
|
depends_on:
|
|
redpanda:
|
|
condition: service_healthy
|
|
entrypoint: /bin/sh
|
|
command:
|
|
- -c
|
|
- |
|
|
rpk topic create cdr.raw -p 3 -r 1 -c compression.type=lz4 -c retention.ms=86400000 --brokers=redpanda:29092 || true
|
|
rpk topic create cdr.raw.dlq -p 1 -r 1 -c retention.ms=604800000 --brokers=redpanda:29092 || true
|
|
restart: "no"
|
|
|
|
# ==========================================
|
|
# Analytics Database: ClickHouse
|
|
# ==========================================
|
|
clickhouse:
|
|
image: clickhouse/clickhouse-server:24.3-alpine
|
|
container_name: clickhouse
|
|
ports:
|
|
- "8123:8123"
|
|
- "9000:9000"
|
|
environment:
|
|
CLICKHOUSE_DB: telecom
|
|
CLICKHOUSE_USER: default
|
|
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
|
volumes:
|
|
- ./docker/clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
|
|
healthcheck:
|
|
test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 10
|
|
start_period: 10s
|
|
|
|
# ==========================================
|
|
# Service 1: Ingestion API Gate
|
|
# ==========================================
|
|
api:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile.api
|
|
container_name: ingestion-api
|
|
ports:
|
|
- "8000:8000"
|
|
environment:
|
|
API_HOST: 0.0.0.0
|
|
API_PORT: 8000
|
|
API_QUEUE_MAX_SIZE: 10000
|
|
WORKER_STATS_URL: http://worker:8001/internal/stats
|
|
KAFKA_BOOTSTRAP_SERVERS: redpanda:29092
|
|
KAFKA_RAW_TOPIC: cdr.raw
|
|
LOG_LEVEL: INFO
|
|
depends_on:
|
|
redpanda:
|
|
condition: service_healthy
|
|
topic-init:
|
|
condition: service_completed_successfully
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 10
|
|
start_period: 5s
|
|
|
|
# ==========================================
|
|
# Service 2: Processing Worker
|
|
# ==========================================
|
|
worker:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile.worker
|
|
container_name: processing-worker
|
|
ports:
|
|
- "8001:8001"
|
|
environment:
|
|
WORKER_INSTANCE: worker-1
|
|
WORKER_STATS_HOST: 0.0.0.0
|
|
WORKER_STATS_PORT: 8001
|
|
KAFKA_BOOTSTRAP_SERVERS: redpanda:29092
|
|
KAFKA_RAW_TOPIC: cdr.raw
|
|
KAFKA_DLQ_TOPIC: cdr.raw.dlq
|
|
KAFKA_CONSUMER_GROUP: cdr-processing-group
|
|
KAFKA_AUTO_OFFSET_RESET: earliest
|
|
CLICKHOUSE_HOST: clickhouse
|
|
CLICKHOUSE_PORT: 8123
|
|
CLICKHOUSE_USER: default
|
|
CLICKHOUSE_PASSWORD: ""
|
|
CLICKHOUSE_DATABASE: telecom
|
|
CLICKHOUSE_TABLE: cdr_records
|
|
BATCH_SIZE: 1000
|
|
BATCH_FLUSH_INTERVAL_SECONDS: 2
|
|
CH_RETRY_MAX_ATTEMPTS: 3
|
|
CH_RETRY_INITIAL_DELAY_MS: 500
|
|
CH_RETRY_BACKOFF_FACTOR: 2.0
|
|
LOG_LEVEL: INFO
|
|
depends_on:
|
|
redpanda:
|
|
condition: service_healthy
|
|
topic-init:
|
|
condition: service_completed_successfully
|
|
clickhouse:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8001/health')"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 10
|
|
start_period: 5s
|