telecom-cdr-pipeline-ai/app/worker/normalizer.py

import json
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, Dict, Optional, Tuple

import phonenumbers

RAT_MAP = {
    1: "3G",
    6: "LTE_4G",
    9: "5G_NR",
}

REQUIRED_FIELDS = [
    "raw_id",
    "device_id",
    "caller_num",
    "svc",
    "rat",
    "epoch_start",
    "epoch_end",
    "up_bytes",
    "down_bytes",
    "lac",
    "ci",
]


@dataclass(frozen=True)
class NormalizedRecord:
    cdr_id: str
    raw_id: str
    msisdn: str
    imsi: str
    service_type: str
    radio_access_type: str
    cell_id: str
    bytes_uploaded: int
    bytes_downloaded: int
    total_bytes: int
    duration_seconds: int
    session_start_time: str
    session_end_time: str
    source_ip: str

    def to_row(self) -> Tuple[Any, ...]:
        """Returns ordered tuple matching ClickHouse cdr_records schema columns."""
        return (
            uuid.UUID(self.cdr_id),
            self.raw_id,
            self.msisdn,
            self.imsi,
            self.service_type,
            self.radio_access_type,
            self.cell_id,
            self.bytes_uploaded,
            self.bytes_downloaded,
            self.total_bytes,
            self.duration_seconds,
            datetime.strptime(self.session_start_time, "%Y-%m-%d %H:%M:%S"),
            datetime.strptime(self.session_end_time, "%Y-%m-%d %H:%M:%S"),
            self.source_ip,
        )


def normalize_phone_number(raw_num: str) -> Optional[str]:
    """
    Normalizes telephone number into E.164 format:
    - Replaces leading '00' or '011' with '+'
    - Preserves existing '+'
    - Validates with phonenumbers library
    """
    if not isinstance(raw_num, str):
        return None

    cleaned = raw_num.strip()
    if cleaned.startswith("00"):
        cleaned = "+" + cleaned[2:]
    elif cleaned.startswith("011"):
        cleaned = "+" + cleaned[3:]
    elif not cleaned.startswith("+"):
        cleaned = "+" + cleaned

    try:
        parsed = phonenumbers.parse(cleaned, None)
        if not (
            phonenumbers.is_valid_number(parsed)
            or phonenumbers.is_possible_number(parsed)
        ):
            return None
        return phonenumbers.format_number(
            parsed, phonenumbers.PhoneNumberFormat.E164
        )
    except Exception:
        return None


def normalize_cdr_payload(
    raw_payload_bytes: bytes,
) -> Tuple[Optional[NormalizedRecord], Optional[str], Optional[str], Optional[str]]:
    """
    Validates and normalizes raw CDR payload.
    Returns:
      (record, failure_reason, failed_field, raw_id)
    If record is None, failure_reason describes why it should be routed to DLQ.
    """
    # 1. Parse JSON
    try:
        if isinstance(raw_payload_bytes, bytes):
            payload_str = raw_payload_bytes.decode("utf-8")
        else:
            payload_str = str(raw_payload_bytes)
        payload: Dict[str, Any] = json.loads(payload_str)
    except Exception:
        return None, "malformed_json", None, None

    if not isinstance(payload, dict):
        return None, "malformed_json", None, None

    raw_id = payload.get("raw_id")
    raw_id_str = str(raw_id) if raw_id is not None else None

    # 2. Check required fields
    for field in REQUIRED_FIELDS:
        if field not in payload or payload[field] is None:
            return None, "missing_required_field", field, raw_id_str

    # 3. Check timestamps (epoch_start <= epoch_end)
    try:
        epoch_start = int(payload["epoch_start"])
        epoch_end = int(payload["epoch_end"])
        if epoch_start > epoch_end:
            return None, "invalid_epoch", "epoch_start", raw_id_str
    except (ValueError, TypeError):
        return None, "invalid_epoch", "epoch_start", raw_id_str

    # 4. Check bytes (must be >= 0)
    try:
        up_bytes = int(payload["up_bytes"])
        down_bytes = int(payload["down_bytes"])
        if up_bytes < 0:
            return None, "normalization_error", "up_bytes", raw_id_str
        if down_bytes < 0:
            return None, "normalization_error", "down_bytes", raw_id_str
    except (ValueError, TypeError):
        return None, "normalization_error", "bytes", raw_id_str

    # 5. Radio Access Type mapping
    rat_raw = payload.get("rat")
    try:
        rat_int = int(rat_raw)
        if rat_int not in RAT_MAP:
            return None, "invalid_rat_value", "rat", raw_id_str
        radio_access_type = RAT_MAP[rat_int]
    except (ValueError, TypeError):
        return None, "invalid_rat_value", "rat", raw_id_str

    # 6. E.164 phone normalization
    caller_num = str(payload["caller_num"])
    msisdn = normalize_phone_number(caller_num)
    if not msisdn:
        return None, "normalization_error", "caller_num", raw_id_str

    # 7. Field mappings and derivations
    imsi = str(payload["device_id"])
    service_type = str(payload["svc"])
    cell_id = f"{payload['lac']}_{payload['ci']}"
    total_bytes = up_bytes + down_bytes
    duration_seconds = epoch_end - epoch_start

    start_dt = datetime.fromtimestamp(epoch_start, tz=timezone.utc).strftime(
        "%Y-%m-%d %H:%M:%S"
    )
    end_dt = datetime.fromtimestamp(epoch_end, tz=timezone.utc).strftime(
        "%Y-%m-%d %H:%M:%S"
    )

    src_ip = str(payload.get("src_ip") or "0.0.0.0")

    # Deterministic UUIDv5 cdr_id from raw_id
    cdr_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, raw_id_str))

    record = NormalizedRecord(
        cdr_id=cdr_id,
        raw_id=raw_id_str,
        msisdn=msisdn,
        imsi=imsi,
        service_type=service_type,
        radio_access_type=radio_access_type,
        cell_id=cell_id,
        bytes_uploaded=up_bytes,
        bytes_downloaded=down_bytes,
        total_bytes=total_bytes,
        duration_seconds=duration_seconds,
        session_start_time=start_dt,
        session_end_time=end_dt,
        source_ip=src_ip,
    )
    return record, None, None, raw_id_str