feat: add pipeline observability and stats

tests/unit/test_batch.py

@@ -58,6 +58,8 @@ def buffered_item(
             partition=0,
             offset=offset,
         ),
+
+        kafka_timestamp_ms=None,
     )
 
 

tests/unit/test_worker_stats.py

@@ -0,0 +1,95 @@
+from datetime import datetime, timezone
+from ipaddress import IPv4Address
+from uuid import uuid4
+
+from core.domain.models import NormalizedCDR
+from core.worker.stats import WorkerRuntimeStats
+
+
+def normalized_cdr() -> NormalizedCDR:
+    now = datetime.now(
+        timezone.utc
+    )
+
+    return NormalizedCDR(
+        cdr_id=uuid4(),
+        raw_id="raw-stats-1",
+
+        msisdn="+19875551234",
+        imsi="310410998877665",
+
+        service_type="4G_DATA",
+        radio_access_type="LTE_4G",
+        cell_id="4021_8812",
+
+        bytes_uploaded=100,
+        bytes_downloaded=200,
+        total_bytes=300,
+        duration_seconds=300,
+
+        session_start_time=now,
+        session_end_time=now,
+
+        source_ip=IPv4Address(
+            "10.240.12.1"
+        ),
+    )
+
+
+def test_worker_stats() -> None:
+    stats = WorkerRuntimeStats()
+
+    stats.record_processed(
+        normalized_cdr()
+    )
+
+    stats.record_dlq(
+        "invalid_rat_value"
+    )
+
+    stats.set_buffer_size(4)
+
+    stats.record_flush()
+
+    snapshot = stats.snapshot()
+
+    assert (
+        snapshot["pipeline"]["total_processed"]
+        == 1
+    )
+
+    assert (
+        snapshot["pipeline"]["total_dlq"]
+        == 1
+    )
+
+    assert (
+        snapshot["pipeline"]["in_flight_buffer"]
+        == 4
+    )
+
+    assert (
+        snapshot["breakdown"]
+        ["by_service_type"]
+        ["4G_DATA"]
+        == 1
+    )
+
+    assert (
+        snapshot["breakdown"]
+        ["by_radio_access_type"]
+        ["LTE_4G"]
+        == 1
+    )
+
+    assert (
+        snapshot["breakdown"]
+        ["dlq_by_reason"]
+        ["invalid_rat_value"]
+        == 1
+    )
+
+    assert (
+        snapshot["pipeline"]["last_flush_at"]
+        is not None
+    )