fix(docker): provision durable topics and canonical ClickHouse schema

app/common/bootstrap.py

@@ -0,0 +1,77 @@
+"""Idempotent infrastructure provisioning; never delete existing data."""
+import re
+from pathlib import Path
+
+from app.common.config import config
+
+SCHEMA = Path(__file__).parents[1] / "worker" / "schema.sql"
+SORT_KEY = "service_type, radio_access_type, cell_id, session_start_time, msisdn, cdr_id"
+
+
+def identifier(value):
+    if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value):
+        raise ValueError(f"Invalid SQL identifier: {value!r}")
+    return value
+
+
+def verify_table(client, database, table):
+    rows = client.query("SELECT engine, sorting_key FROM system.tables WHERE database={db:String} AND name={name:String}",
+                        parameters={"db": database, "name": table}).result_rows
+    if not rows or rows[0][0] != "ReplacingMergeTree" or rows[0][1] != SORT_KEY:
+        raise RuntimeError("Incompatible ClickHouse table. Use the explicit assessment reset in README; no data was deleted.")
+    expected = re.findall(r"^    (\w+)\s+(UUID|String|LowCardinality\(String\)|UInt64|UInt32|DateTime64\(3\)|DateTime|IPv4)[,\s]", SCHEMA.read_text(), re.M)
+    actual = client.query("SELECT name, type FROM system.columns WHERE database={db:String} AND table={name:String} ORDER BY position",
+                          parameters={"db": database, "name": table}).result_rows
+    if list(map(tuple, actual)) != expected:
+        raise RuntimeError("Incompatible ClickHouse columns; use the explicit assessment reset in README.")
+    ddl = client.command(f"SHOW CREATE TABLE {identifier(database)}.{identifier(table)}")
+    if "ReplacingMergeTree(processed_at)" not in ddl or "toYYYYMM(session_start_time)" not in ddl or "TTL session_start_time + toIntervalDay(90)" not in ddl:
+        raise RuntimeError("Incompatible ClickHouse engine/partition/TTL; use the explicit assessment reset in README.")
+
+
+def provision_database():
+    import clickhouse_connect
+    ch = config.clickhouse
+    database, table = identifier(ch.database), identifier(ch.table)
+    client = clickhouse_connect.get_client(host=ch.host, port=ch.port, username=ch.user,
+                                           password=ch.password, database="default")
+    try:
+        existing = client.query("SELECT count() FROM system.tables WHERE database={db:String} AND name={name:String}",
+                                parameters={"db": database, "name": table}).first_row[0]
+        if not existing:
+            for statement in SCHEMA.read_text().format(database=database, table=table).split(";"):
+                if statement.strip():
+                    client.command(statement)
+        verify_table(client, database, table)
+    finally:
+        client.close()
+
+
+def provision_topics():
+    from confluent_kafka.admin import AdminClient, NewTopic, ConfigResource, ResourceType
+    admin = AdminClient({"bootstrap.servers": config.kafka.bootstrap_servers})
+    required = {config.kafka.raw_topic: (3, {"retention.ms": "86400000", "compression.type": "lz4"}),
+                config.kafka.dlq_topic: (1, {"retention.ms": "604800000"})}
+    if len(required) != 2:
+        raise ValueError("Raw and DLQ topic names must differ")
+    existing = admin.list_topics(timeout=10).topics
+    missing = [NewTopic(name, num_partitions=count, replication_factor=1, config=settings)
+               for name, (count, settings) in required.items() if name not in existing]
+    for future in admin.create_topics(missing).values() if missing else []:
+        future.result(timeout=15)
+    metadata = admin.list_topics(timeout=10)
+    for name, (count, settings) in required.items():
+        topic = metadata.topics[name]
+        if topic.error or len(topic.partitions) != count or any(len(p.replicas) != 1 for p in topic.partitions.values()):
+            raise RuntimeError(f"Incompatible topic partitions/replication: {name}")
+        resource = ConfigResource(ResourceType.TOPIC, name)
+        actual = admin.describe_configs([resource])[resource].result(timeout=10)
+        for key, value in settings.items():
+            if actual[key].value != value:
+                raise RuntimeError(f"Topic {name} requires {key}={value}; found {actual[key].value}")
+
+
+if __name__ == "__main__":
+    provision_topics()
+    provision_database()
+    print("Topics and ClickHouse schema verified")

app/worker/schema.sql

@@ -0,0 +1,41 @@
+-- ClickHouse Initialization Script for Telecom CDR Ingestion Pipeline
+-- Database: telecom
+-- Table: {database}.{table}
+
+CREATE DATABASE IF NOT EXISTS {database};
+
+CREATE TABLE IF NOT EXISTS {database}.{table} (
+    -- Identifiers
+    cdr_id              UUID,
+    raw_id              String,
+
+    -- Subscriber
+    msisdn              String,
+    imsi                String,
+
+    -- Session metadata
+    service_type        LowCardinality(String),
+    radio_access_type   LowCardinality(String),
+    cell_id             LowCardinality(String),
+
+    -- Traffic
+    bytes_uploaded      UInt64,
+    bytes_downloaded    UInt64,
+    total_bytes         UInt64,
+    duration_seconds    UInt32,
+
+    -- Timing
+    session_start_time  DateTime,
+    session_end_time    DateTime,
+
+    -- Network
+    source_ip           IPv4,
+
+    -- Audit
+    processed_at        DateTime64(3) DEFAULT now()
+
+) ENGINE = ReplacingMergeTree(processed_at)
+PARTITION BY toYYYYMM(session_start_time)
+ORDER BY (service_type, radio_access_type, cell_id, session_start_time, msisdn, cdr_id)
+TTL session_start_time + INTERVAL 90 DAY
+SETTINGS index_granularity = 8192;