refactor(observability): restructure into service-owned packages

tests/integration/test_ingestion_api.py

@@ -5,6 +5,7 @@ import pytest
 from fastapi.testclient import TestClient
 
 from core.api.main import create_app
+from core.api.observability.stats import WorkerStatsUnavailable
 from core.api.publisher import (
     IngestionQueueFull,
     KafkaPublishError,
@@ -327,3 +328,31 @@ def test_successful_request_increments_api_metric(
     )
 
 
+def test_stats_endpoint_when_worker_unavailable(
+    publisher: FakePublisher,
+) -> None:
+    class FailingWorkerStatsClient:
+        async def get_stats(self) -> dict[str, Any]:
+            raise WorkerStatsUnavailable(
+                "Worker stats endpoint unreachable"
+            )
+
+        async def close(self) -> None:
+            pass
+
+    app = create_app(
+        publisher=publisher,
+        worker_stats_client=FailingWorkerStatsClient(),
+    )
+
+    with TestClient(app) as test_client:
+        response = test_client.get("/stats")
+
+        assert response.status_code == 503
+
+        body = response.json()
+        assert body["error"] == "worker_stats_unavailable"
+        assert "Worker stats endpoint unreachable" in body["detail"]
+
+
+

tests/unit/test_api_observability.py

@@ -0,0 +1,135 @@
+import httpx2
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from prometheus_client import CONTENT_TYPE_LATEST
+
+from core.api.observability.metrics import ApiMetrics
+from core.api.observability.routes import create_observability_router
+from core.api.observability.stats import (
+    ApiRuntimeStats,
+    HttpWorkerStatsClient,
+    WorkerStatsUnavailable,
+)
+
+
+@pytest.mark.anyio
+async def test_http_worker_stats_client_success() -> None:
+    async def fake_handler(request: httpx2.Request) -> httpx2.Response:
+        assert request.url == "http://test-worker/internal/stats"
+        return httpx2.Response(
+            status_code=200,
+            json={
+                "pipeline": {
+                    "total_processed": 5,
+                    "total_dlq": 1,
+                    "in_flight_buffer": 2,
+                    "last_flush_at": "2026-09-22T12:00:00Z",
+                },
+                "breakdown": {
+                    "by_service_type": {"4G_DATA": 5},
+                    "by_radio_access_type": {"LTE_4G": 5},
+                    "dlq_by_reason": {"timeout": 1},
+                },
+            },
+        )
+
+    transport = httpx2.MockTransport(fake_handler)
+    client = HttpWorkerStatsClient(
+        url="http://test-worker/internal/stats",
+        timeout_seconds=2.0,
+    )
+    # Inject mock transport
+    client._client = httpx2.AsyncClient(transport=transport)
+
+    stats = await client.get_stats()
+    assert stats["pipeline"]["total_processed"] == 5
+    await client.close()
+
+
+@pytest.mark.anyio
+async def test_http_worker_stats_client_error() -> None:
+    async def failing_handler(request: httpx2.Request) -> httpx2.Response:
+        return httpx2.Response(status_code=500)
+
+    transport = httpx2.MockTransport(failing_handler)
+    client = HttpWorkerStatsClient(
+        url="http://test-worker/internal/stats",
+        timeout_seconds=2.0,
+    )
+    client._client = httpx2.AsyncClient(transport=transport)
+
+    with pytest.raises(WorkerStatsUnavailable) as exc_info:
+        await client.get_stats()
+
+    assert "Unable to retrieve worker stats" in str(exc_info.value)
+    await client.close()
+
+
+def test_api_observability_router_metrics() -> None:
+    metrics = ApiMetrics()
+    stats = ApiRuntimeStats()
+
+    class DummyWorkerStatsProvider:
+        async def get_stats(self) -> dict:
+            return {}
+
+        async def close(self) -> None:
+            pass
+
+    app = FastAPI()
+    router = create_observability_router(
+        metrics=metrics,
+        stats=stats,
+        worker_stats_provider=DummyWorkerStatsProvider(),
+    )
+    app.include_router(router)
+
+    client = TestClient(app)
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    assert CONTENT_TYPE_LATEST in response.headers["content-type"]
+    assert "cdr_api_received_total" in response.text
+
+
+def test_api_observability_router_stats_aggregation() -> None:
+    metrics = ApiMetrics()
+    stats = ApiRuntimeStats()
+    stats.record_received()
+    stats.record_received()
+
+    class FakeProvider:
+        async def get_stats(self) -> dict:
+            return {
+                "pipeline": {
+                    "total_processed": 15,
+                    "total_dlq": 3,
+                    "in_flight_buffer": 0,
+                    "last_flush_at": "2026-09-22T10:00:00Z",
+                },
+                "breakdown": {
+                    "by_service_type": {"4G_DATA": 15},
+                    "by_radio_access_type": {"LTE_4G": 15},
+                    "dlq_by_reason": {},
+                },
+            }
+
+        async def close(self) -> None:
+            pass
+
+    app = FastAPI()
+    router = create_observability_router(
+        metrics=metrics,
+        stats=stats,
+        worker_stats_provider=FakeProvider(),
+    )
+    app.include_router(router)
+
+    client = TestClient(app)
+    response = client.get("/stats")
+    assert response.status_code == 200
+    body = response.json()
+    assert body["pipeline"]["total_received"] == 2
+    assert body["pipeline"]["total_processed"] == 15
+    assert body["pipeline"]["total_dlq"] == 3
+

tests/unit/test_worker_observability.py

@@ -0,0 +1,109 @@
+from datetime import datetime, timezone
+from ipaddress import IPv4Address
+from uuid import uuid4
+
+from fastapi.testclient import TestClient
+from prometheus_client import CONTENT_TYPE_LATEST
+
+from core.domain.models import NormalizedCDR
+from core.worker.observability.metrics import WorkerMetrics
+from core.worker.observability.server import create_worker_observability_app
+from core.worker.observability.stats import WorkerRuntimeStats
+
+
+def create_sample_cdr() -> NormalizedCDR:
+    now = datetime.now(timezone.utc)
+    return NormalizedCDR(
+        cdr_id=uuid4(),
+        raw_id="raw-obs-1",
+        msisdn="+19875551234",
+        imsi="310410998877665",
+        service_type="4G_DATA",
+        radio_access_type="LTE_4G",
+        cell_id="4021_8812",
+        bytes_uploaded=100,
+        bytes_downloaded=200,
+        total_bytes=300,
+        duration_seconds=300,
+        session_start_time=now,
+        session_end_time=now,
+        source_ip=IPv4Address("10.240.12.1"),
+    )
+
+
+def test_worker_observability_health_endpoint() -> None:
+    metrics = WorkerMetrics()
+    stats = WorkerRuntimeStats()
+    app = create_worker_observability_app(metrics=metrics, stats=stats)
+    client = TestClient(app)
+
+    response = client.get("/health")
+    assert response.status_code == 200
+    assert response.json() == {"status": "ok"}
+
+
+def test_worker_observability_metrics_endpoint() -> None:
+    metrics = WorkerMetrics()
+    stats = WorkerRuntimeStats()
+    app = create_worker_observability_app(metrics=metrics, stats=stats)
+    client = TestClient(app)
+
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    assert CONTENT_TYPE_LATEST in response.headers["content-type"]
+    assert "cdr_processed_total" in response.text
+    assert "cdr_dlq_total" in response.text
+
+
+def test_worker_observability_internal_stats_endpoint() -> None:
+    metrics = WorkerMetrics()
+    stats = WorkerRuntimeStats()
+    app = create_worker_observability_app(metrics=metrics, stats=stats)
+    client = TestClient(app)
+
+    response = client.get("/internal/stats")
+    assert response.status_code == 200
+    body = response.json()
+    assert "pipeline" in body
+    assert "breakdown" in body
+    assert body["pipeline"]["total_processed"] == 0
+    assert body["pipeline"]["total_dlq"] == 0
+
+
+def test_shared_instance_behavior_between_processing_and_observability() -> None:
+    # Processing and observability must share the identical metrics and stats instances
+    metrics = WorkerMetrics()
+    stats = WorkerRuntimeStats()
+    app = create_worker_observability_app(metrics=metrics, stats=stats)
+    client = TestClient(app)
+
+    # Simulate worker processing activity updating the instances
+    cdr = create_sample_cdr()
+    metrics.record_processed(cdr)
+    stats.record_processed(cdr)
+
+    metrics.record_dlq("invalid_rat_value")
+    stats.record_dlq("invalid_rat_value")
+
+    metrics.set_buffer_size(5)
+    stats.set_buffer_size(5)
+
+    # Verify that the observability server sees the exact same updates
+    stats_response = client.get("/internal/stats")
+    assert stats_response.status_code == 200
+    stats_body = stats_response.json()
+
+    assert stats_body["pipeline"]["total_processed"] == 1
+    assert stats_body["pipeline"]["total_dlq"] == 1
+    assert stats_body["pipeline"]["in_flight_buffer"] == 5
+    assert stats_body["breakdown"]["by_service_type"]["4G_DATA"] == 1
+    assert stats_body["breakdown"]["by_radio_access_type"]["LTE_4G"] == 1
+    assert stats_body["breakdown"]["dlq_by_reason"]["invalid_rat_value"] == 1
+
+    # Verify metrics endpoint output contains the recorded metrics
+    metrics_response = client.get("/metrics")
+    assert metrics_response.status_code == 200
+    assert 'cdr_processed_total{radio_access_type="LTE_4G",service_type="4G_DATA"} 1.0' in metrics_response.text
+    assert 'cdr_dlq_total{reason="invalid_rat_value"} 1.0' in metrics_response.text
+    assert 'cdr_in_flight_buffer_size 5.0' in metrics_response.text
+

tests/unit/test_worker_stats.py

@@ -3,7 +3,7 @@ from ipaddress import IPv4Address
 from uuid import uuid4
 
 from core.domain.models import NormalizedCDR
-from core.worker.stats import WorkerRuntimeStats
+from core.worker.observability.stats import WorkerRuntimeStats
 
 
 def normalized_cdr() -> NormalizedCDR: