feat: add pipeline observability and stats

tests/integration/test_ingestion_api.py

@@ -38,6 +38,35 @@ class FakePublisher:
         )
 
 
+class FakeWorkerStatsClient:
+    async def get_stats(
+        self,
+    ) -> dict[str, Any]:
+        return {
+            "pipeline": {
+                "total_processed": 10,
+                "total_dlq": 2,
+                "in_flight_buffer": 3,
+                "last_flush_at":
+                    "2026-09-20T12:00:00Z",
+            },
+            "breakdown": {
+                "by_service_type": {
+                    "4G_DATA": 10,
+                },
+                "by_radio_access_type": {
+                    "LTE_4G": 10,
+                },
+                "dlq_by_reason": {
+                    "invalid_rat_value": 2,
+                },
+            },
+        }
+
+    async def close(self) -> None:
+        pass
+
+
 @pytest.fixture
 def publisher() -> FakePublisher:
     return FakePublisher()
@@ -49,6 +78,7 @@ def client(
 ):
     app = create_app(
         publisher=publisher,
+        worker_stats_client=FakeWorkerStatsClient(),
     )
 
     with TestClient(app) as test_client:
@@ -226,3 +256,74 @@ def test_kafka_failure_returns_503(
     assert response.status_code == 503
 
     assert response.json()["error"] == "kafka_unavailable"
+
+
+def test_metrics_endpoint(
+    client: TestClient,
+) -> None:
+    response = client.get(
+        "/metrics"
+    )
+
+    assert response.status_code == 200
+
+    assert (
+        "cdr_api_received_total"
+        in response.text
+    )
+
+
+def test_stats_endpoint(
+    client: TestClient,
+) -> None:
+    response = client.get(
+        "/stats"
+    )
+
+    assert response.status_code == 200
+
+    body = response.json()
+
+    assert (
+        body["pipeline"]["total_processed"]
+        == 10
+    )
+
+    assert (
+        body["pipeline"]["total_dlq"]
+        == 2
+    )
+
+    assert (
+        body["pipeline"]["in_flight_buffer"]
+        == 3
+    )
+
+    assert (
+        body["breakdown"]
+        ["by_service_type"]
+        ["4G_DATA"]
+        == 10
+    )
+
+
+def test_successful_request_increments_api_metric(
+    client: TestClient,
+) -> None:
+    response = client.post(
+        "/api/v1/cdr/raw",
+        json=valid_payload(),
+    )
+
+    assert response.status_code == 202
+
+    metrics = client.get(
+        "/metrics"
+    )
+
+    assert (
+        'cdr_api_received_total 1.0'
+        in metrics.text
+    )
+
+

tests/unit/test_batch.py

@@ -58,6 +58,8 @@ def buffered_item(
             partition=0,
             offset=offset,
         ),
+
+        kafka_timestamp_ms=None,
     )
 
 

tests/unit/test_worker_stats.py

@@ -0,0 +1,95 @@
+from datetime import datetime, timezone
+from ipaddress import IPv4Address
+from uuid import uuid4
+
+from core.domain.models import NormalizedCDR
+from core.worker.stats import WorkerRuntimeStats
+
+
+def normalized_cdr() -> NormalizedCDR:
+    now = datetime.now(
+        timezone.utc
+    )
+
+    return NormalizedCDR(
+        cdr_id=uuid4(),
+        raw_id="raw-stats-1",
+
+        msisdn="+19875551234",
+        imsi="310410998877665",
+
+        service_type="4G_DATA",
+        radio_access_type="LTE_4G",
+        cell_id="4021_8812",
+
+        bytes_uploaded=100,
+        bytes_downloaded=200,
+        total_bytes=300,
+        duration_seconds=300,
+
+        session_start_time=now,
+        session_end_time=now,
+
+        source_ip=IPv4Address(
+            "10.240.12.1"
+        ),
+    )
+
+
+def test_worker_stats() -> None:
+    stats = WorkerRuntimeStats()
+
+    stats.record_processed(
+        normalized_cdr()
+    )
+
+    stats.record_dlq(
+        "invalid_rat_value"
+    )
+
+    stats.set_buffer_size(4)
+
+    stats.record_flush()
+
+    snapshot = stats.snapshot()
+
+    assert (
+        snapshot["pipeline"]["total_processed"]
+        == 1
+    )
+
+    assert (
+        snapshot["pipeline"]["total_dlq"]
+        == 1
+    )
+
+    assert (
+        snapshot["pipeline"]["in_flight_buffer"]
+        == 4
+    )
+
+    assert (
+        snapshot["breakdown"]
+        ["by_service_type"]
+        ["4G_DATA"]
+        == 1
+    )
+
+    assert (
+        snapshot["breakdown"]
+        ["by_radio_access_type"]
+        ["LTE_4G"]
+        == 1
+    )
+
+    assert (
+        snapshot["breakdown"]
+        ["dlq_by_reason"]
+        ["invalid_rat_value"]
+        == 1
+    )
+
+    assert (
+        snapshot["pipeline"]["last_flush_at"]
+        is not None
+    )