commit f86b68a00b7c6d342a0c2d926b2c518aa6f66b99 Author: Amirhossein Khalili Date: Sun Sep 20 11:13:58 2026 +0330 chore: bootstrap project structure diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b934fdf --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ + +# Virtual environment +.venv/ + +# Environment variables +.env + +# IDEs +.idea/ +.vscode/ + +# OS +.DS_Store +Thumbs.db + +# Coverage +.coverage +htmlcov/ + +# Logs +*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..cfe4985 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# Telecom CDR Pipeline + +High-throughput telecom Call Detail Record ingestion and analytics pipeline. + +## Architecture + +The system will consist of: + +1. FastAPI ingestion API +2. Kafka-compatible Redpanda message broker +3. Processing and normalization worker +4. ClickHouse analytics database + +Data flow: + +Network Element → Ingestion API → Redpanda → Worker → ClickHouse + +Malformed or unprocessable records are routed by the worker to a Dead Letter Queue. + +## Development Status + +Currently under development. \ No newline at end of file diff --git a/infrastructure/clickhouse/init.sql b/infrastructure/clickhouse/init.sql new file mode 100644 index 0000000..a83056d --- /dev/null +++ b/infrastructure/clickhouse/init.sql @@ -0,0 +1,38 @@ +CREATE DATABASE IF NOT EXISTS telecom; + +CREATE TABLE IF NOT EXISTS telecom.cdr_records +( + cdr_id UUID, + raw_id String, + + msisdn String, + imsi String, + + service_type LowCardinality(String), + radio_access_type LowCardinality(String), + cell_id LowCardinality(String), + + bytes_uploaded UInt64, + bytes_downloaded UInt64, + total_bytes UInt64, + duration_seconds UInt32, + + session_start_time DateTime, + session_end_time DateTime, + + source_ip IPv4, + + processed_at DateTime64(3) DEFAULT now() +) +ENGINE = ReplacingMergeTree(processed_at) +PARTITION BY toYYYYMM(session_start_time) +ORDER BY ( + service_type, + radio_access_type, + cell_id, + session_start_time, + msisdn, + cdr_id +) +TTL session_start_time + INTERVAL 90 DAY +SETTINGS index_granularity = 8192; diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c395c89 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "telecom-cdr-pipeline" +version = "0.1.0" +description = "High-throughput telecom CDR ingestion and analytics pipeline" +requires-python = ">=3.11" +dependencies = [] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", +] + +[build-system] +requires = ["setuptools>=75.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/src/core/__init__.py b/src/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_project_setup.py b/tests/unit/test_project_setup.py new file mode 100644 index 0000000..293f4c1 --- /dev/null +++ b/tests/unit/test_project_setup.py @@ -0,0 +1,2 @@ +def test_project_setup() -> None: + assert True \ No newline at end of file