chore: bootstrap project structure
This commit is contained in:
0
.env.sample
Normal file
0
.env.sample
Normal file
26
.gitignore
vendored
Normal file
26
.gitignore
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
|
||||
# Virtual environment
|
||||
.venv/
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
|
||||
# IDEs
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Coverage
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
22
README.md
Normal file
22
README.md
Normal file
@@ -0,0 +1,22 @@
|
||||
# Telecom CDR Pipeline
|
||||
|
||||
High-throughput telecom Call Detail Record ingestion and analytics pipeline.
|
||||
|
||||
## Architecture
|
||||
|
||||
The system will consist of:
|
||||
|
||||
1. FastAPI ingestion API
|
||||
2. Kafka-compatible Redpanda message broker
|
||||
3. Processing and normalization worker
|
||||
4. ClickHouse analytics database
|
||||
|
||||
Data flow:
|
||||
|
||||
Network Element → Ingestion API → Redpanda → Worker → ClickHouse
|
||||
|
||||
Malformed or unprocessable records are routed by the worker to a Dead Letter Queue.
|
||||
|
||||
## Development Status
|
||||
|
||||
Currently under development.
|
||||
38
infrastructure/clickhouse/init.sql
Normal file
38
infrastructure/clickhouse/init.sql
Normal file
@@ -0,0 +1,38 @@
|
||||
CREATE DATABASE IF NOT EXISTS telecom;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS telecom.cdr_records
|
||||
(
|
||||
cdr_id UUID,
|
||||
raw_id String,
|
||||
|
||||
msisdn String,
|
||||
imsi String,
|
||||
|
||||
service_type LowCardinality(String),
|
||||
radio_access_type LowCardinality(String),
|
||||
cell_id LowCardinality(String),
|
||||
|
||||
bytes_uploaded UInt64,
|
||||
bytes_downloaded UInt64,
|
||||
total_bytes UInt64,
|
||||
duration_seconds UInt32,
|
||||
|
||||
session_start_time DateTime,
|
||||
session_end_time DateTime,
|
||||
|
||||
source_ip IPv4,
|
||||
|
||||
processed_at DateTime64(3) DEFAULT now()
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(processed_at)
|
||||
PARTITION BY toYYYYMM(session_start_time)
|
||||
ORDER BY (
|
||||
service_type,
|
||||
radio_access_type,
|
||||
cell_id,
|
||||
session_start_time,
|
||||
msisdn,
|
||||
cdr_id
|
||||
)
|
||||
TTL session_start_time + INTERVAL 90 DAY
|
||||
SETTINGS index_granularity = 8192;
|
||||
22
pyproject.toml
Normal file
22
pyproject.toml
Normal file
@@ -0,0 +1,22 @@
|
||||
[project]
|
||||
name = "telecom-cdr-pipeline"
|
||||
version = "0.1.0"
|
||||
description = "High-throughput telecom CDR ingestion and analytics pipeline"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = []
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=75.0"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["src"]
|
||||
0
src/core/__init__.py
Normal file
0
src/core/__init__.py
Normal file
2
tests/unit/test_project_setup.py
Normal file
2
tests/unit/test_project_setup.py
Normal file
@@ -0,0 +1,2 @@
|
||||
def test_project_setup() -> None:
|
||||
assert True
|
||||
Reference in New Issue
Block a user