feat(worker): complete Phase 3 - processing worker with normalization, micro-batching, and at-least-once offset management

app/worker/normalizer.py

@@ -0,0 +1,200 @@
+import json
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional, Tuple
+
+import phonenumbers
+
+RAT_MAP = {
+    1: "3G",
+    6: "LTE_4G",
+    9: "5G_NR",
+}
+
+REQUIRED_FIELDS = [
+    "raw_id",
+    "device_id",
+    "caller_num",
+    "svc",
+    "rat",
+    "epoch_start",
+    "epoch_end",
+    "up_bytes",
+    "down_bytes",
+    "lac",
+    "ci",
+]
+
+
+@dataclass(frozen=True)
+class NormalizedRecord:
+    cdr_id: str
+    raw_id: str
+    msisdn: str
+    imsi: str
+    service_type: str
+    radio_access_type: str
+    cell_id: str
+    bytes_uploaded: int
+    bytes_downloaded: int
+    total_bytes: int
+    duration_seconds: int
+    session_start_time: str
+    session_end_time: str
+    source_ip: str
+
+    def to_row(self) -> Tuple[Any, ...]:
+        """Returns ordered tuple matching ClickHouse cdr_records schema columns."""
+        return (
+            uuid.UUID(self.cdr_id),
+            self.raw_id,
+            self.msisdn,
+            self.imsi,
+            self.service_type,
+            self.radio_access_type,
+            self.cell_id,
+            self.bytes_uploaded,
+            self.bytes_downloaded,
+            self.total_bytes,
+            self.duration_seconds,
+            datetime.strptime(self.session_start_time, "%Y-%m-%d %H:%M:%S"),
+            datetime.strptime(self.session_end_time, "%Y-%m-%d %H:%M:%S"),
+            self.source_ip,
+        )
+
+
+def normalize_phone_number(raw_num: str) -> Optional[str]:
+    """
+    Normalizes telephone number into E.164 format:
+    - Replaces leading '00' or '011' with '+'
+    - Preserves existing '+'
+    - Validates with phonenumbers library
+    """
+    if not isinstance(raw_num, str):
+        return None
+
+    cleaned = raw_num.strip()
+    if cleaned.startswith("00"):
+        cleaned = "+" + cleaned[2:]
+    elif cleaned.startswith("011"):
+        cleaned = "+" + cleaned[3:]
+    elif not cleaned.startswith("+"):
+        cleaned = "+" + cleaned
+
+    try:
+        parsed = phonenumbers.parse(cleaned, None)
+        if not (
+            phonenumbers.is_valid_number(parsed)
+            or phonenumbers.is_possible_number(parsed)
+        ):
+            return None
+        return phonenumbers.format_number(
+            parsed, phonenumbers.PhoneNumberFormat.E164
+        )
+    except Exception:
+        return None
+
+
+def normalize_cdr_payload(
+    raw_payload_bytes: bytes,
+) -> Tuple[Optional[NormalizedRecord], Optional[str], Optional[str], Optional[str]]:
+    """
+    Validates and normalizes raw CDR payload.
+    Returns:
+      (record, failure_reason, failed_field, raw_id)
+    If record is None, failure_reason describes why it should be routed to DLQ.
+    """
+    # 1. Parse JSON
+    try:
+        if isinstance(raw_payload_bytes, bytes):
+            payload_str = raw_payload_bytes.decode("utf-8")
+        else:
+            payload_str = str(raw_payload_bytes)
+        payload: Dict[str, Any] = json.loads(payload_str)
+    except Exception:
+        return None, "malformed_json", None, None
+
+    if not isinstance(payload, dict):
+        return None, "malformed_json", None, None
+
+    raw_id = payload.get("raw_id")
+    raw_id_str = str(raw_id) if raw_id is not None else None
+
+    # 2. Check required fields
+    for field in REQUIRED_FIELDS:
+        if field not in payload or payload[field] is None:
+            return None, "missing_required_field", field, raw_id_str
+
+    # 3. Check timestamps (epoch_start <= epoch_end)
+    try:
+        epoch_start = int(payload["epoch_start"])
+        epoch_end = int(payload["epoch_end"])
+        if epoch_start > epoch_end:
+            return None, "invalid_epoch", "epoch_start", raw_id_str
+    except (ValueError, TypeError):
+        return None, "invalid_epoch", "epoch_start", raw_id_str
+
+    # 4. Check bytes (must be >= 0)
+    try:
+        up_bytes = int(payload["up_bytes"])
+        down_bytes = int(payload["down_bytes"])
+        if up_bytes < 0:
+            return None, "normalization_error", "up_bytes", raw_id_str
+        if down_bytes < 0:
+            return None, "normalization_error", "down_bytes", raw_id_str
+    except (ValueError, TypeError):
+        return None, "normalization_error", "bytes", raw_id_str
+
+    # 5. Radio Access Type mapping
+    rat_raw = payload.get("rat")
+    try:
+        rat_int = int(rat_raw)
+        if rat_int not in RAT_MAP:
+            return None, "invalid_rat_value", "rat", raw_id_str
+        radio_access_type = RAT_MAP[rat_int]
+    except (ValueError, TypeError):
+        return None, "invalid_rat_value", "rat", raw_id_str
+
+    # 6. E.164 phone normalization
+    caller_num = str(payload["caller_num"])
+    msisdn = normalize_phone_number(caller_num)
+    if not msisdn:
+        return None, "normalization_error", "caller_num", raw_id_str
+
+    # 7. Field mappings and derivations
+    imsi = str(payload["device_id"])
+    service_type = str(payload["svc"])
+    cell_id = f"{payload['lac']}_{payload['ci']}"
+    total_bytes = up_bytes + down_bytes
+    duration_seconds = epoch_end - epoch_start
+
+    start_dt = datetime.fromtimestamp(epoch_start, tz=timezone.utc).strftime(
+        "%Y-%m-%d %H:%M:%S"
+    )
+    end_dt = datetime.fromtimestamp(epoch_end, tz=timezone.utc).strftime(
+        "%Y-%m-%d %H:%M:%S"
+    )
+
+    src_ip = str(payload.get("src_ip") or "0.0.0.0")
+
+    # Deterministic UUIDv5 cdr_id from raw_id
+    cdr_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, raw_id_str))
+
+    record = NormalizedRecord(
+        cdr_id=cdr_id,
+        raw_id=raw_id_str,
+        msisdn=msisdn,
+        imsi=imsi,
+        service_type=service_type,
+        radio_access_type=radio_access_type,
+        cell_id=cell_id,
+        bytes_uploaded=up_bytes,
+        bytes_downloaded=down_bytes,
+        total_bytes=total_bytes,
+        duration_seconds=duration_seconds,
+        session_start_time=start_dt,
+        session_end_time=end_dt,
+        source_ip=src_ip,
+    )
+    return record, None, None, raw_id_str