infra: add clickhouse storage and schema initialization

.env.sample

@@ -19,4 +19,19 @@ KAFKA_DLQ_TOPIC_PARTITIONS=1
 KAFKA_DLQ_TOPIC_REPLICATION_FACTOR=1
 KAFKA_DLQ_TOPIC_RETENTION_MS=604800000
 
-KAFKA_CONSUMER_GROUP=cdr-processing-workers
+KAFKA_CONSUMER_GROUP=cdr-processing-workers
+
+
+# =========================
+# ClickHouse
+# =========================
+
+CLICKHOUSE_VERSION=26.8.7.19
+
+CLICKHOUSE_HOST=clickhouse
+CLICKHOUSE_PORT=8123
+CLICKHOUSE_HTTP_HOST_PORT=8123
+
+CLICKHOUSE_DATABASE=telecom
+CLICKHOUSE_USER=telecom
+CLICKHOUSE_PASSWORD=telecom_dev_password

docker-compose.yml

@@ -2,7 +2,7 @@ name: telecom-cdr-pipeline
 
 services:
   redpanda:
-    image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-v26.2.3}
+    image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-latest}
 
     container_name: telecom-redpanda
 
@@ -49,7 +49,7 @@ services:
       start_period: 10s
 
   redpanda-init:
-    image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-v26.2.3}
+    image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-latest}
 
     depends_on:
       redpanda:
@@ -92,5 +92,41 @@ services:
 
         echo "Kafka topics initialized successfully."
 
+  clickhouse:
+    image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-latest}
+
+    container_name: telecom-clickhouse
+
+    environment:
+      CLICKHOUSE_DB: ${CLICKHOUSE_DATABASE}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+      CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS: 1
+
+    ports:
+      - "${CLICKHOUSE_HTTP_HOST_PORT:-8123}:8123"
+
+    volumes:
+      - clickhouse-data:/var/lib/clickhouse
+      - ./infrastructure/clickhouse/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro
+
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "clickhouse-client --user \"$$CLICKHOUSE_USER\" --password \"$$CLICKHOUSE_PASSWORD\" --query \"SELECT 1\""
+        ]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
 volumes:
-  redpanda-data:
+  redpanda-data:
+  clickhouse-data:

infrastructure/clickhouse/init.sql

@@ -2,31 +2,39 @@ CREATE DATABASE IF NOT EXISTS telecom;
 
 CREATE TABLE IF NOT EXISTS telecom.cdr_records
 (
+    -- Identifiers
     cdr_id UUID,
     raw_id String,
 
+    -- Subscriber
     msisdn String,
     imsi String,
 
+    -- Session metadata
     service_type LowCardinality(String),
     radio_access_type LowCardinality(String),
     cell_id LowCardinality(String),
 
+    -- Traffic
     bytes_uploaded UInt64,
     bytes_downloaded UInt64,
     total_bytes UInt64,
     duration_seconds UInt32,
 
+    -- Timing
     session_start_time DateTime,
     session_end_time DateTime,
 
+    -- Network
     source_ip IPv4,
 
-    processed_at DateTime64(3) DEFAULT now()
+    -- Audit
+    processed_at DateTime64(3) DEFAULT now64(3)
 )
 ENGINE = ReplacingMergeTree(processed_at)
 PARTITION BY toYYYYMM(session_start_time)
-ORDER BY (
+ORDER BY
+(
     service_type,
     radio_access_type,
     cell_id,