diff --git a/src/core/api/main.py b/src/core/api/main.py index 6ae26f3..a5c5dd3 100644 --- a/src/core/api/main.py +++ b/src/core/api/main.py @@ -1,21 +1,18 @@ from contextlib import asynccontextmanager from fastapi import FastAPI -from fastapi.responses import JSONResponse, Response - -from prometheus_client import CONTENT_TYPE_LATEST from core.api.kafka_publisher import KafkaPublisher -from core.api.publisher import Publisher -from core.api.routes import router -from core.config import get_settings -from core.api.metrics import ApiMetrics -from core.api.stats import ( +from core.api.observability.metrics import ApiMetrics +from core.api.observability.routes import create_observability_router +from core.api.observability.stats import ( ApiRuntimeStats, HttpWorkerStatsClient, WorkerStatsProvider, - WorkerStatsUnavailable, ) +from core.api.publisher import Publisher +from core.api.routes import router +from core.config import get_settings def create_app( @@ -27,6 +24,15 @@ def create_app( api_metrics = ApiMetrics() api_stats = ApiRuntimeStats() + active_worker_stats_client = ( + worker_stats_client + if worker_stats_client is not None + else HttpWorkerStatsClient( + url=settings.worker_stats_url, + timeout_seconds=settings.worker_stats_timeout_seconds, + ) + ) + @asynccontextmanager async def lifespan(app: FastAPI): active_publisher = publisher @@ -36,20 +42,6 @@ def create_app( settings ) - active_worker_stats_client = ( - worker_stats_client - ) - - if active_worker_stats_client is None: - active_worker_stats_client = ( - HttpWorkerStatsClient( - url=settings.worker_stats_url, - timeout_seconds=( - settings.worker_stats_timeout_seconds - ), - ) - ) - app.state.publisher = active_publisher app.state.worker_stats_client = ( @@ -75,73 +67,27 @@ def create_app( app.state.api_metrics = api_metrics app.state.api_stats = api_stats + observability_router = create_observability_router( + metrics=api_metrics, + stats=api_stats, + worker_stats_provider=active_worker_stats_client, + ) + app.include_router( router, prefix="/api/v1", ) + app.include_router( + observability_router, + ) + @app.get("/health") async def health() -> dict[str, str]: return { "status": "ok", } - @app.get( - "/metrics", - include_in_schema=False, - ) - async def metrics() -> Response: - return Response( - content=api_metrics.render(), - media_type=CONTENT_TYPE_LATEST, - ) - - @app.get("/stats") - async def stats(): - try: - worker_stats = await ( - app.state - .worker_stats_client - .get_stats() - ) - - except WorkerStatsUnavailable as exc: - return JSONResponse( - status_code=503, - content={ - "error": "worker_stats_unavailable", - "detail": str(exc), - }, - ) - - api_snapshot = api_stats.snapshot() - - worker_pipeline = ( - worker_stats["pipeline"] - ) - - return { - "pipeline": { - "total_received": - api_snapshot["total_received"], - - "total_processed": - worker_pipeline["total_processed"], - - "total_dlq": - worker_pipeline["total_dlq"], - - "in_flight_buffer": - worker_pipeline["in_flight_buffer"], - - "last_flush_at": - worker_pipeline["last_flush_at"], - }, - - "breakdown": - worker_stats["breakdown"], - } - return app diff --git a/src/core/api/observability/__init__.py b/src/core/api/observability/__init__.py new file mode 100644 index 0000000..33ee0c4 --- /dev/null +++ b/src/core/api/observability/__init__.py @@ -0,0 +1,2 @@ +"""API observability package.""" + diff --git a/src/core/api/metrics.py b/src/core/api/observability/metrics.py similarity index 100% rename from src/core/api/metrics.py rename to src/core/api/observability/metrics.py diff --git a/src/core/api/observability/routes.py b/src/core/api/observability/routes.py new file mode 100644 index 0000000..0037530 --- /dev/null +++ b/src/core/api/observability/routes.py @@ -0,0 +1,59 @@ +from typing import Any + +from fastapi import APIRouter, Response +from fastapi.responses import JSONResponse +from prometheus_client import CONTENT_TYPE_LATEST + +from core.api.observability.metrics import ApiMetrics +from core.api.observability.stats import ( + ApiRuntimeStats, + WorkerStatsProvider, + WorkerStatsUnavailable, +) + + +def create_observability_router( + metrics: ApiMetrics, + stats: ApiRuntimeStats, + worker_stats_provider: WorkerStatsProvider, +) -> APIRouter: + router = APIRouter() + + @router.get( + "/metrics", + include_in_schema=False, + ) + async def metrics_endpoint() -> Response: + return Response( + content=metrics.render(), + media_type=CONTENT_TYPE_LATEST, + ) + + @router.get("/stats") + async def stats_endpoint(): + try: + worker_stats = await worker_stats_provider.get_stats() + except WorkerStatsUnavailable as exc: + return JSONResponse( + status_code=503, + content={ + "error": "worker_stats_unavailable", + "detail": str(exc), + }, + ) + + api_snapshot = stats.snapshot() + worker_pipeline = worker_stats["pipeline"] + + return { + "pipeline": { + "total_received": api_snapshot["total_received"], + "total_processed": worker_pipeline["total_processed"], + "total_dlq": worker_pipeline["total_dlq"], + "in_flight_buffer": worker_pipeline["in_flight_buffer"], + "last_flush_at": worker_pipeline["last_flush_at"], + }, + "breakdown": worker_stats["breakdown"], + } + + return router diff --git a/src/core/api/stats.py b/src/core/api/observability/stats.py similarity index 100% rename from src/core/api/stats.py rename to src/core/api/observability/stats.py diff --git a/src/core/worker/consumer.py b/src/core/worker/consumer.py index d84a861..2c7cad1 100644 --- a/src/core/worker/consumer.py +++ b/src/core/worker/consumer.py @@ -27,12 +27,12 @@ from core.worker.dlq import ( DLQPublishError, DLQPublisher, ) -from core.worker.metrics import WorkerMetrics +from core.worker.observability.metrics import WorkerMetrics from core.worker.processor import ( decode_payload, process_payload, ) -from core.worker.stats import WorkerRuntimeStats +from core.worker.observability.stats import WorkerRuntimeStats logger = logging.getLogger(__name__) diff --git a/src/core/worker/main.py b/src/core/worker/main.py index a3c2701..64fd24d 100644 --- a/src/core/worker/main.py +++ b/src/core/worker/main.py @@ -3,9 +3,9 @@ import signal from core.config import get_settings from core.worker.consumer import ProcessingWorker -from core.worker.metrics import WorkerMetrics -from core.worker.observability import WorkerObservabilityServer -from core.worker.stats import WorkerRuntimeStats +from core.worker.observability.metrics import WorkerMetrics +from core.worker.observability.server import WorkerObservabilityServer +from core.worker.observability.stats import WorkerRuntimeStats logging.basicConfig( level=logging.INFO, diff --git a/src/core/worker/observability/__init__.py b/src/core/worker/observability/__init__.py new file mode 100644 index 0000000..df04664 --- /dev/null +++ b/src/core/worker/observability/__init__.py @@ -0,0 +1,2 @@ +"""Worker observability package.""" + diff --git a/src/core/worker/metrics.py b/src/core/worker/observability/metrics.py similarity index 100% rename from src/core/worker/metrics.py rename to src/core/worker/observability/metrics.py diff --git a/src/core/worker/observability.py b/src/core/worker/observability/server.py similarity index 94% rename from src/core/worker/observability.py rename to src/core/worker/observability/server.py index 83ee53e..cc03a53 100644 --- a/src/core/worker/observability.py +++ b/src/core/worker/observability/server.py @@ -7,8 +7,8 @@ from fastapi.responses import Response from prometheus_client import CONTENT_TYPE_LATEST from core.config import Settings -from core.worker.metrics import WorkerMetrics -from core.worker.stats import WorkerRuntimeStats +from core.worker.observability.metrics import WorkerMetrics +from core.worker.observability.stats import WorkerRuntimeStats logger = logging.getLogger(__name__) diff --git a/src/core/worker/stats.py b/src/core/worker/observability/stats.py similarity index 100% rename from src/core/worker/stats.py rename to src/core/worker/observability/stats.py diff --git a/tests/integration/test_ingestion_api.py b/tests/integration/test_ingestion_api.py index fc7bb5a..df1fbda 100644 --- a/tests/integration/test_ingestion_api.py +++ b/tests/integration/test_ingestion_api.py @@ -5,6 +5,7 @@ import pytest from fastapi.testclient import TestClient from core.api.main import create_app +from core.api.observability.stats import WorkerStatsUnavailable from core.api.publisher import ( IngestionQueueFull, KafkaPublishError, @@ -327,3 +328,31 @@ def test_successful_request_increments_api_metric( ) +def test_stats_endpoint_when_worker_unavailable( + publisher: FakePublisher, +) -> None: + class FailingWorkerStatsClient: + async def get_stats(self) -> dict[str, Any]: + raise WorkerStatsUnavailable( + "Worker stats endpoint unreachable" + ) + + async def close(self) -> None: + pass + + app = create_app( + publisher=publisher, + worker_stats_client=FailingWorkerStatsClient(), + ) + + with TestClient(app) as test_client: + response = test_client.get("/stats") + + assert response.status_code == 503 + + body = response.json() + assert body["error"] == "worker_stats_unavailable" + assert "Worker stats endpoint unreachable" in body["detail"] + + + diff --git a/tests/unit/test_api_observability.py b/tests/unit/test_api_observability.py new file mode 100644 index 0000000..080bb98 --- /dev/null +++ b/tests/unit/test_api_observability.py @@ -0,0 +1,135 @@ +import httpx2 +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from prometheus_client import CONTENT_TYPE_LATEST + +from core.api.observability.metrics import ApiMetrics +from core.api.observability.routes import create_observability_router +from core.api.observability.stats import ( + ApiRuntimeStats, + HttpWorkerStatsClient, + WorkerStatsUnavailable, +) + + +@pytest.mark.anyio +async def test_http_worker_stats_client_success() -> None: + async def fake_handler(request: httpx2.Request) -> httpx2.Response: + assert request.url == "http://test-worker/internal/stats" + return httpx2.Response( + status_code=200, + json={ + "pipeline": { + "total_processed": 5, + "total_dlq": 1, + "in_flight_buffer": 2, + "last_flush_at": "2026-09-22T12:00:00Z", + }, + "breakdown": { + "by_service_type": {"4G_DATA": 5}, + "by_radio_access_type": {"LTE_4G": 5}, + "dlq_by_reason": {"timeout": 1}, + }, + }, + ) + + transport = httpx2.MockTransport(fake_handler) + client = HttpWorkerStatsClient( + url="http://test-worker/internal/stats", + timeout_seconds=2.0, + ) + # Inject mock transport + client._client = httpx2.AsyncClient(transport=transport) + + stats = await client.get_stats() + assert stats["pipeline"]["total_processed"] == 5 + await client.close() + + +@pytest.mark.anyio +async def test_http_worker_stats_client_error() -> None: + async def failing_handler(request: httpx2.Request) -> httpx2.Response: + return httpx2.Response(status_code=500) + + transport = httpx2.MockTransport(failing_handler) + client = HttpWorkerStatsClient( + url="http://test-worker/internal/stats", + timeout_seconds=2.0, + ) + client._client = httpx2.AsyncClient(transport=transport) + + with pytest.raises(WorkerStatsUnavailable) as exc_info: + await client.get_stats() + + assert "Unable to retrieve worker stats" in str(exc_info.value) + await client.close() + + +def test_api_observability_router_metrics() -> None: + metrics = ApiMetrics() + stats = ApiRuntimeStats() + + class DummyWorkerStatsProvider: + async def get_stats(self) -> dict: + return {} + + async def close(self) -> None: + pass + + app = FastAPI() + router = create_observability_router( + metrics=metrics, + stats=stats, + worker_stats_provider=DummyWorkerStatsProvider(), + ) + app.include_router(router) + + client = TestClient(app) + response = client.get("/metrics") + assert response.status_code == 200 + assert CONTENT_TYPE_LATEST in response.headers["content-type"] + assert "cdr_api_received_total" in response.text + + +def test_api_observability_router_stats_aggregation() -> None: + metrics = ApiMetrics() + stats = ApiRuntimeStats() + stats.record_received() + stats.record_received() + + class FakeProvider: + async def get_stats(self) -> dict: + return { + "pipeline": { + "total_processed": 15, + "total_dlq": 3, + "in_flight_buffer": 0, + "last_flush_at": "2026-09-22T10:00:00Z", + }, + "breakdown": { + "by_service_type": {"4G_DATA": 15}, + "by_radio_access_type": {"LTE_4G": 15}, + "dlq_by_reason": {}, + }, + } + + async def close(self) -> None: + pass + + app = FastAPI() + router = create_observability_router( + metrics=metrics, + stats=stats, + worker_stats_provider=FakeProvider(), + ) + app.include_router(router) + + client = TestClient(app) + response = client.get("/stats") + assert response.status_code == 200 + body = response.json() + assert body["pipeline"]["total_received"] == 2 + assert body["pipeline"]["total_processed"] == 15 + assert body["pipeline"]["total_dlq"] == 3 + diff --git a/tests/unit/test_worker_observability.py b/tests/unit/test_worker_observability.py new file mode 100644 index 0000000..24f7c3d --- /dev/null +++ b/tests/unit/test_worker_observability.py @@ -0,0 +1,109 @@ +from datetime import datetime, timezone +from ipaddress import IPv4Address +from uuid import uuid4 + +from fastapi.testclient import TestClient +from prometheus_client import CONTENT_TYPE_LATEST + +from core.domain.models import NormalizedCDR +from core.worker.observability.metrics import WorkerMetrics +from core.worker.observability.server import create_worker_observability_app +from core.worker.observability.stats import WorkerRuntimeStats + + +def create_sample_cdr() -> NormalizedCDR: + now = datetime.now(timezone.utc) + return NormalizedCDR( + cdr_id=uuid4(), + raw_id="raw-obs-1", + msisdn="+19875551234", + imsi="310410998877665", + service_type="4G_DATA", + radio_access_type="LTE_4G", + cell_id="4021_8812", + bytes_uploaded=100, + bytes_downloaded=200, + total_bytes=300, + duration_seconds=300, + session_start_time=now, + session_end_time=now, + source_ip=IPv4Address("10.240.12.1"), + ) + + +def test_worker_observability_health_endpoint() -> None: + metrics = WorkerMetrics() + stats = WorkerRuntimeStats() + app = create_worker_observability_app(metrics=metrics, stats=stats) + client = TestClient(app) + + response = client.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_worker_observability_metrics_endpoint() -> None: + metrics = WorkerMetrics() + stats = WorkerRuntimeStats() + app = create_worker_observability_app(metrics=metrics, stats=stats) + client = TestClient(app) + + response = client.get("/metrics") + assert response.status_code == 200 + assert CONTENT_TYPE_LATEST in response.headers["content-type"] + assert "cdr_processed_total" in response.text + assert "cdr_dlq_total" in response.text + + +def test_worker_observability_internal_stats_endpoint() -> None: + metrics = WorkerMetrics() + stats = WorkerRuntimeStats() + app = create_worker_observability_app(metrics=metrics, stats=stats) + client = TestClient(app) + + response = client.get("/internal/stats") + assert response.status_code == 200 + body = response.json() + assert "pipeline" in body + assert "breakdown" in body + assert body["pipeline"]["total_processed"] == 0 + assert body["pipeline"]["total_dlq"] == 0 + + +def test_shared_instance_behavior_between_processing_and_observability() -> None: + # Processing and observability must share the identical metrics and stats instances + metrics = WorkerMetrics() + stats = WorkerRuntimeStats() + app = create_worker_observability_app(metrics=metrics, stats=stats) + client = TestClient(app) + + # Simulate worker processing activity updating the instances + cdr = create_sample_cdr() + metrics.record_processed(cdr) + stats.record_processed(cdr) + + metrics.record_dlq("invalid_rat_value") + stats.record_dlq("invalid_rat_value") + + metrics.set_buffer_size(5) + stats.set_buffer_size(5) + + # Verify that the observability server sees the exact same updates + stats_response = client.get("/internal/stats") + assert stats_response.status_code == 200 + stats_body = stats_response.json() + + assert stats_body["pipeline"]["total_processed"] == 1 + assert stats_body["pipeline"]["total_dlq"] == 1 + assert stats_body["pipeline"]["in_flight_buffer"] == 5 + assert stats_body["breakdown"]["by_service_type"]["4G_DATA"] == 1 + assert stats_body["breakdown"]["by_radio_access_type"]["LTE_4G"] == 1 + assert stats_body["breakdown"]["dlq_by_reason"]["invalid_rat_value"] == 1 + + # Verify metrics endpoint output contains the recorded metrics + metrics_response = client.get("/metrics") + assert metrics_response.status_code == 200 + assert 'cdr_processed_total{radio_access_type="LTE_4G",service_type="4G_DATA"} 1.0' in metrics_response.text + assert 'cdr_dlq_total{reason="invalid_rat_value"} 1.0' in metrics_response.text + assert 'cdr_in_flight_buffer_size 5.0' in metrics_response.text + diff --git a/tests/unit/test_worker_stats.py b/tests/unit/test_worker_stats.py index 5ef8d29..ab2deb2 100644 --- a/tests/unit/test_worker_stats.py +++ b/tests/unit/test_worker_stats.py @@ -3,7 +3,7 @@ from ipaddress import IPv4Address from uuid import uuid4 from core.domain.models import NormalizedCDR -from core.worker.stats import WorkerRuntimeStats +from core.worker.observability.stats import WorkerRuntimeStats def normalized_cdr() -> NormalizedCDR: