import json import uuid from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, Dict, Optional, Tuple import phonenumbers RAT_MAP = { 1: "3G", 6: "LTE_4G", 9: "5G_NR", } REQUIRED_FIELDS = [ "raw_id", "device_id", "caller_num", "svc", "rat", "epoch_start", "epoch_end", "up_bytes", "down_bytes", "lac", "ci", ] @dataclass(frozen=True) class NormalizedRecord: cdr_id: str raw_id: str msisdn: str imsi: str service_type: str radio_access_type: str cell_id: str bytes_uploaded: int bytes_downloaded: int total_bytes: int duration_seconds: int session_start_time: str session_end_time: str source_ip: str def to_row(self) -> Tuple[Any, ...]: """Returns ordered tuple matching ClickHouse cdr_records schema columns.""" return ( uuid.UUID(self.cdr_id), self.raw_id, self.msisdn, self.imsi, self.service_type, self.radio_access_type, self.cell_id, self.bytes_uploaded, self.bytes_downloaded, self.total_bytes, self.duration_seconds, datetime.strptime(self.session_start_time, "%Y-%m-%d %H:%M:%S"), datetime.strptime(self.session_end_time, "%Y-%m-%d %H:%M:%S"), self.source_ip, ) def normalize_phone_number(raw_num: str) -> Optional[str]: """ Normalizes telephone number into E.164 format: - Replaces leading '00' or '011' with '+' - Preserves existing '+' - Validates with phonenumbers library """ if not isinstance(raw_num, str): return None cleaned = raw_num.strip() if cleaned.startswith("00"): cleaned = "+" + cleaned[2:] elif cleaned.startswith("011"): cleaned = "+" + cleaned[3:] elif not cleaned.startswith("+"): cleaned = "+" + cleaned try: parsed = phonenumbers.parse(cleaned, None) if not ( phonenumbers.is_valid_number(parsed) or phonenumbers.is_possible_number(parsed) ): return None return phonenumbers.format_number( parsed, phonenumbers.PhoneNumberFormat.E164 ) except Exception: return None def normalize_cdr_payload( raw_payload_bytes: bytes, ) -> Tuple[Optional[NormalizedRecord], Optional[str], Optional[str], Optional[str]]: """ Validates and normalizes raw CDR payload. Returns: (record, failure_reason, failed_field, raw_id) If record is None, failure_reason describes why it should be routed to DLQ. """ # 1. Parse JSON try: if isinstance(raw_payload_bytes, bytes): payload_str = raw_payload_bytes.decode("utf-8") else: payload_str = str(raw_payload_bytes) payload: Dict[str, Any] = json.loads(payload_str) except Exception: return None, "malformed_json", None, None if not isinstance(payload, dict): return None, "malformed_json", None, None raw_id = payload.get("raw_id") raw_id_str = str(raw_id) if raw_id is not None else None # 2. Check required fields for field in REQUIRED_FIELDS: if field not in payload or payload[field] is None: return None, "missing_required_field", field, raw_id_str # 3. Check timestamps (epoch_start <= epoch_end) try: epoch_start = int(payload["epoch_start"]) epoch_end = int(payload["epoch_end"]) if epoch_start > epoch_end: return None, "invalid_epoch", "epoch_start", raw_id_str except (ValueError, TypeError): return None, "invalid_epoch", "epoch_start", raw_id_str # 4. Check bytes (must be >= 0) try: up_bytes = int(payload["up_bytes"]) down_bytes = int(payload["down_bytes"]) if up_bytes < 0: return None, "normalization_error", "up_bytes", raw_id_str if down_bytes < 0: return None, "normalization_error", "down_bytes", raw_id_str except (ValueError, TypeError): return None, "normalization_error", "bytes", raw_id_str # 5. Radio Access Type mapping rat_raw = payload.get("rat") try: rat_int = int(rat_raw) if rat_int not in RAT_MAP: return None, "invalid_rat_value", "rat", raw_id_str radio_access_type = RAT_MAP[rat_int] except (ValueError, TypeError): return None, "invalid_rat_value", "rat", raw_id_str # 6. E.164 phone normalization caller_num = str(payload["caller_num"]) msisdn = normalize_phone_number(caller_num) if not msisdn: return None, "normalization_error", "caller_num", raw_id_str # 7. Field mappings and derivations imsi = str(payload["device_id"]) service_type = str(payload["svc"]) cell_id = f"{payload['lac']}_{payload['ci']}" total_bytes = up_bytes + down_bytes duration_seconds = epoch_end - epoch_start start_dt = datetime.fromtimestamp(epoch_start, tz=timezone.utc).strftime( "%Y-%m-%d %H:%M:%S" ) end_dt = datetime.fromtimestamp(epoch_end, tz=timezone.utc).strftime( "%Y-%m-%d %H:%M:%S" ) src_ip = str(payload.get("src_ip") or "0.0.0.0") # Deterministic UUIDv5 cdr_id from raw_id cdr_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, raw_id_str)) record = NormalizedRecord( cdr_id=cdr_id, raw_id=raw_id_str, msisdn=msisdn, imsi=imsi, service_type=service_type, radio_access_type=radio_access_type, cell_id=cell_id, bytes_uploaded=up_bytes, bytes_downloaded=down_bytes, total_bytes=total_bytes, duration_seconds=duration_seconds, session_start_time=start_dt, session_end_time=end_dt, source_ip=src_ip, ) return record, None, None, raw_id_str