78 lines
4.2 KiB
Python
78 lines
4.2 KiB
Python
"""Idempotent infrastructure provisioning; never delete existing data."""
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from app.common.config import config
|
|
|
|
SCHEMA = Path(__file__).parents[1] / "worker" / "schema.sql"
|
|
SORT_KEY = "service_type, radio_access_type, cell_id, session_start_time, msisdn, cdr_id"
|
|
|
|
|
|
def identifier(value):
|
|
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value):
|
|
raise ValueError(f"Invalid SQL identifier: {value!r}")
|
|
return value
|
|
|
|
|
|
def verify_table(client, database, table):
|
|
rows = client.query("SELECT engine, sorting_key FROM system.tables WHERE database={db:String} AND name={name:String}",
|
|
parameters={"db": database, "name": table}).result_rows
|
|
if not rows or rows[0][0] != "ReplacingMergeTree" or rows[0][1] != SORT_KEY:
|
|
raise RuntimeError("Incompatible ClickHouse table. Use the explicit assessment reset in README; no data was deleted.")
|
|
expected = re.findall(r"^ (\w+)\s+(UUID|String|LowCardinality\(String\)|UInt64|UInt32|DateTime64\(3\)|DateTime|IPv4)[,\s]", SCHEMA.read_text(), re.M)
|
|
actual = client.query("SELECT name, type FROM system.columns WHERE database={db:String} AND table={name:String} ORDER BY position",
|
|
parameters={"db": database, "name": table}).result_rows
|
|
if list(map(tuple, actual)) != expected:
|
|
raise RuntimeError("Incompatible ClickHouse columns; use the explicit assessment reset in README.")
|
|
ddl = client.command(f"SHOW CREATE TABLE {identifier(database)}.{identifier(table)}")
|
|
if "ReplacingMergeTree(processed_at)" not in ddl or "toYYYYMM(session_start_time)" not in ddl or "TTL session_start_time + toIntervalDay(90)" not in ddl:
|
|
raise RuntimeError("Incompatible ClickHouse engine/partition/TTL; use the explicit assessment reset in README.")
|
|
|
|
|
|
def provision_database():
|
|
import clickhouse_connect
|
|
ch = config.clickhouse
|
|
database, table = identifier(ch.database), identifier(ch.table)
|
|
client = clickhouse_connect.get_client(host=ch.host, port=ch.port, username=ch.user,
|
|
password=ch.password, database="default")
|
|
try:
|
|
existing = client.query("SELECT count() FROM system.tables WHERE database={db:String} AND name={name:String}",
|
|
parameters={"db": database, "name": table}).first_row[0]
|
|
if not existing:
|
|
for statement in SCHEMA.read_text().format(database=database, table=table).split(";"):
|
|
if statement.strip():
|
|
client.command(statement)
|
|
verify_table(client, database, table)
|
|
finally:
|
|
client.close()
|
|
|
|
|
|
def provision_topics():
|
|
from confluent_kafka.admin import AdminClient, NewTopic, ConfigResource, ResourceType
|
|
admin = AdminClient({"bootstrap.servers": config.kafka.bootstrap_servers})
|
|
required = {config.kafka.raw_topic: (3, {"retention.ms": "86400000", "compression.type": "lz4"}),
|
|
config.kafka.dlq_topic: (1, {"retention.ms": "604800000"})}
|
|
if len(required) != 2:
|
|
raise ValueError("Raw and DLQ topic names must differ")
|
|
existing = admin.list_topics(timeout=10).topics
|
|
missing = [NewTopic(name, num_partitions=count, replication_factor=1, config=settings)
|
|
for name, (count, settings) in required.items() if name not in existing]
|
|
for future in admin.create_topics(missing).values() if missing else []:
|
|
future.result(timeout=15)
|
|
metadata = admin.list_topics(timeout=10)
|
|
for name, (count, settings) in required.items():
|
|
topic = metadata.topics[name]
|
|
if topic.error or len(topic.partitions) != count or any(len(p.replicas) != 1 for p in topic.partitions.values()):
|
|
raise RuntimeError(f"Incompatible topic partitions/replication: {name}")
|
|
resource = ConfigResource(ResourceType.TOPIC, name)
|
|
actual = admin.describe_configs([resource])[resource].result(timeout=10)
|
|
for key, value in settings.items():
|
|
if actual[key].value != value:
|
|
raise RuntimeError(f"Topic {name} requires {key}={value}; found {actual[key].value}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
provision_topics()
|
|
provision_database()
|
|
print("Topics and ClickHouse schema verified")
|