chore: bootstrap project structure
This commit is contained in:
0
.env.sample
Normal file
0
.env.sample
Normal file
26
.gitignore
vendored
Normal file
26
.gitignore
vendored
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*.egg-info/
|
||||||
|
.pytest_cache/
|
||||||
|
|
||||||
|
# Virtual environment
|
||||||
|
.venv/
|
||||||
|
|
||||||
|
# Environment variables
|
||||||
|
.env
|
||||||
|
|
||||||
|
# IDEs
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
# OS
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Coverage
|
||||||
|
.coverage
|
||||||
|
htmlcov/
|
||||||
|
|
||||||
|
# Logs
|
||||||
|
*.log
|
||||||
22
README.md
Normal file
22
README.md
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
# Telecom CDR Pipeline
|
||||||
|
|
||||||
|
High-throughput telecom Call Detail Record ingestion and analytics pipeline.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
The system will consist of:
|
||||||
|
|
||||||
|
1. FastAPI ingestion API
|
||||||
|
2. Kafka-compatible Redpanda message broker
|
||||||
|
3. Processing and normalization worker
|
||||||
|
4. ClickHouse analytics database
|
||||||
|
|
||||||
|
Data flow:
|
||||||
|
|
||||||
|
Network Element → Ingestion API → Redpanda → Worker → ClickHouse
|
||||||
|
|
||||||
|
Malformed or unprocessable records are routed by the worker to a Dead Letter Queue.
|
||||||
|
|
||||||
|
## Development Status
|
||||||
|
|
||||||
|
Currently under development.
|
||||||
38
infrastructure/clickhouse/init.sql
Normal file
38
infrastructure/clickhouse/init.sql
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
CREATE DATABASE IF NOT EXISTS telecom;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS telecom.cdr_records
|
||||||
|
(
|
||||||
|
cdr_id UUID,
|
||||||
|
raw_id String,
|
||||||
|
|
||||||
|
msisdn String,
|
||||||
|
imsi String,
|
||||||
|
|
||||||
|
service_type LowCardinality(String),
|
||||||
|
radio_access_type LowCardinality(String),
|
||||||
|
cell_id LowCardinality(String),
|
||||||
|
|
||||||
|
bytes_uploaded UInt64,
|
||||||
|
bytes_downloaded UInt64,
|
||||||
|
total_bytes UInt64,
|
||||||
|
duration_seconds UInt32,
|
||||||
|
|
||||||
|
session_start_time DateTime,
|
||||||
|
session_end_time DateTime,
|
||||||
|
|
||||||
|
source_ip IPv4,
|
||||||
|
|
||||||
|
processed_at DateTime64(3) DEFAULT now()
|
||||||
|
)
|
||||||
|
ENGINE = ReplacingMergeTree(processed_at)
|
||||||
|
PARTITION BY toYYYYMM(session_start_time)
|
||||||
|
ORDER BY (
|
||||||
|
service_type,
|
||||||
|
radio_access_type,
|
||||||
|
cell_id,
|
||||||
|
session_start_time,
|
||||||
|
msisdn,
|
||||||
|
cdr_id
|
||||||
|
)
|
||||||
|
TTL session_start_time + INTERVAL 90 DAY
|
||||||
|
SETTINGS index_granularity = 8192;
|
||||||
22
pyproject.toml
Normal file
22
pyproject.toml
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
[project]
|
||||||
|
name = "telecom-cdr-pipeline"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "High-throughput telecom CDR ingestion and analytics pipeline"
|
||||||
|
requires-python = ">=3.11"
|
||||||
|
dependencies = []
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
dev = [
|
||||||
|
"pytest>=8.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=75.0"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[tool.setuptools.packages.find]
|
||||||
|
where = ["src"]
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
pythonpath = ["src"]
|
||||||
0
src/core/__init__.py
Normal file
0
src/core/__init__.py
Normal file
2
tests/unit/test_project_setup.py
Normal file
2
tests/unit/test_project_setup.py
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
def test_project_setup() -> None:
|
||||||
|
assert True
|
||||||
Reference in New Issue
Block a user