diff --git a/.env.sample b/.env.sample index a4a3d92..688967a 100644 --- a/.env.sample +++ b/.env.sample @@ -19,4 +19,19 @@ KAFKA_DLQ_TOPIC_PARTITIONS=1 KAFKA_DLQ_TOPIC_REPLICATION_FACTOR=1 KAFKA_DLQ_TOPIC_RETENTION_MS=604800000 -KAFKA_CONSUMER_GROUP=cdr-processing-workers \ No newline at end of file +KAFKA_CONSUMER_GROUP=cdr-processing-workers + + +# ========================= +# ClickHouse +# ========================= + +CLICKHOUSE_VERSION=26.8.7.19 + +CLICKHOUSE_HOST=clickhouse +CLICKHOUSE_PORT=8123 +CLICKHOUSE_HTTP_HOST_PORT=8123 + +CLICKHOUSE_DATABASE=telecom +CLICKHOUSE_USER=telecom +CLICKHOUSE_PASSWORD=telecom_dev_password diff --git a/docker-compose.yml b/docker-compose.yml index 9ad126c..fcf19d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ name: telecom-cdr-pipeline services: redpanda: - image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-v26.2.3} + image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-latest} container_name: telecom-redpanda @@ -49,7 +49,7 @@ services: start_period: 10s redpanda-init: - image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-v26.2.3} + image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-latest} depends_on: redpanda: @@ -92,5 +92,41 @@ services: echo "Kafka topics initialized successfully." + clickhouse: + image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-latest} + + container_name: telecom-clickhouse + + environment: + CLICKHOUSE_DB: ${CLICKHOUSE_DATABASE} + CLICKHOUSE_USER: ${CLICKHOUSE_USER} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS: 1 + + ports: + - "${CLICKHOUSE_HTTP_HOST_PORT:-8123}:8123" + + volumes: + - clickhouse-data:/var/lib/clickhouse + - ./infrastructure/clickhouse/init.sql:/docker-entrypoint-initdb.d/001-init.sql:ro + + ulimits: + nofile: + soft: 262144 + hard: 262144 + + healthcheck: + test: + [ + "CMD-SHELL", + "clickhouse-client --user \"$$CLICKHOUSE_USER\" --password \"$$CLICKHOUSE_PASSWORD\" --query \"SELECT 1\"" + ] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + volumes: - redpanda-data: \ No newline at end of file + redpanda-data: + clickhouse-data: diff --git a/infrastructure/clickhouse/init.sql b/infrastructure/clickhouse/init.sql index a83056d..987926e 100644 --- a/infrastructure/clickhouse/init.sql +++ b/infrastructure/clickhouse/init.sql @@ -2,31 +2,39 @@ CREATE DATABASE IF NOT EXISTS telecom; CREATE TABLE IF NOT EXISTS telecom.cdr_records ( + -- Identifiers cdr_id UUID, raw_id String, + -- Subscriber msisdn String, imsi String, + -- Session metadata service_type LowCardinality(String), radio_access_type LowCardinality(String), cell_id LowCardinality(String), + -- Traffic bytes_uploaded UInt64, bytes_downloaded UInt64, total_bytes UInt64, duration_seconds UInt32, + -- Timing session_start_time DateTime, session_end_time DateTime, + -- Network source_ip IPv4, - processed_at DateTime64(3) DEFAULT now() + -- Audit + processed_at DateTime64(3) DEFAULT now64(3) ) ENGINE = ReplacingMergeTree(processed_at) PARTITION BY toYYYYMM(session_start_time) -ORDER BY ( +ORDER BY +( service_type, radio_access_type, cell_id,