feat: add pipeline observability and stats

src/core/api/main.py

@@ -1,27 +1,61 @@
 from contextlib import asynccontextmanager
 
 from fastapi import FastAPI
+from fastapi.responses import JSONResponse, Response
+
+from prometheus_client import CONTENT_TYPE_LATEST
 
 from core.api.kafka_publisher import KafkaPublisher
 from core.api.publisher import Publisher
 from core.api.routes import router
 from core.config import get_settings
+from core.api.metrics import ApiMetrics
+from core.api.stats import (
+    ApiRuntimeStats,
+    HttpWorkerStatsClient,
+    WorkerStatsProvider,
+    WorkerStatsUnavailable,
+)
 
 
 def create_app(
     publisher: Publisher | None = None,
+    worker_stats_client: WorkerStatsProvider | None = None,
 ) -> FastAPI:
+    settings = get_settings()
+
+    api_metrics = ApiMetrics()
+    api_stats = ApiRuntimeStats()
+
     @asynccontextmanager
     async def lifespan(app: FastAPI):
         active_publisher = publisher
 
         if active_publisher is None:
             active_publisher = KafkaPublisher(
-                get_settings()
+                settings
+            )
+
+        active_worker_stats_client = (
+            worker_stats_client
+        )
+
+        if active_worker_stats_client is None:
+            active_worker_stats_client = (
+                HttpWorkerStatsClient(
+                    url=settings.worker_stats_url,
+                    timeout_seconds=(
+                        settings.worker_stats_timeout_seconds
+                    ),
+                )
             )
 
         app.state.publisher = active_publisher
 
+        app.state.worker_stats_client = (
+            active_worker_stats_client
+        )
+
         await active_publisher.start()
 
         try:
@@ -30,12 +64,17 @@ def create_app(
         finally:
             await active_publisher.stop()
 
+            await active_worker_stats_client.close()
+
     app = FastAPI(
         title="Telecom CDR Ingestion API",
         version="0.1.0",
         lifespan=lifespan,
     )
 
+    app.state.api_metrics = api_metrics
+    app.state.api_stats = api_stats
+
     app.include_router(
         router,
         prefix="/api/v1",
@@ -47,6 +86,62 @@ def create_app(
             "status": "ok",
         }
 
+    @app.get(
+        "/metrics",
+        include_in_schema=False,
+    )
+    async def metrics() -> Response:
+        return Response(
+            content=api_metrics.render(),
+            media_type=CONTENT_TYPE_LATEST,
+        )
+
+    @app.get("/stats")
+    async def stats():
+        try:
+            worker_stats = await (
+                app.state
+                .worker_stats_client
+                .get_stats()
+            )
+
+        except WorkerStatsUnavailable as exc:
+            return JSONResponse(
+                status_code=503,
+                content={
+                    "error": "worker_stats_unavailable",
+                    "detail": str(exc),
+                },
+            )
+
+        api_snapshot = api_stats.snapshot()
+
+        worker_pipeline = (
+            worker_stats["pipeline"]
+        )
+
+        return {
+            "pipeline": {
+                "total_received":
+                    api_snapshot["total_received"],
+
+                "total_processed":
+                    worker_pipeline["total_processed"],
+
+                "total_dlq":
+                    worker_pipeline["total_dlq"],
+
+                "in_flight_buffer":
+                    worker_pipeline["in_flight_buffer"],
+
+                "last_flush_at":
+                    worker_pipeline["last_flush_at"],
+            },
+
+            "breakdown":
+                worker_stats["breakdown"],
+        }
+
     return app