import asyncio import json import urllib.request import urllib.error from typing import Any, Dict from app.api.metrics import api_metrics from app.common.logging import setup_logger logger = setup_logger("ingestion-api") def _sync_fetch_worker_stats(url: str, timeout: float = 2.0) -> Dict[str, Any]: req = urllib.request.Request(url, headers={"Accept": "application/json"}) with urllib.request.urlopen(req, timeout=timeout) as response: if response.status == 200: content = response.read().decode("utf-8") return json.loads(content) raise ValueError(f"Worker returned status {response.status}") async def get_aggregated_pipeline_stats(worker_stats_url: str) -> Dict[str, Any]: """ Combines local Ingestion API statistics with remote runtime statistics from the Processing Worker. """ total_received = api_metrics.total_received try: loop = asyncio.get_running_loop() worker_stats = await loop.run_in_executor( None, _sync_fetch_worker_stats, worker_stats_url ) pipeline_data = worker_stats.get("pipeline", {}) breakdown_data = worker_stats.get("breakdown", {}) return { "pipeline": { "total_received": total_received, "total_processed": pipeline_data.get("total_processed", 0), "total_dlq": pipeline_data.get("total_dlq", 0), "in_flight_buffer": pipeline_data.get("in_flight_buffer", 0), "last_flush_at": pipeline_data.get("last_flush_at"), }, "breakdown": { "by_service_type": breakdown_data.get("by_service_type", {}), "by_radio_access_type": breakdown_data.get("by_radio_access_type", {}), "dlq_by_reason": breakdown_data.get("dlq_by_reason", {}), }, } except Exception as e: logger.warning( f"Unable to fetch worker stats from {worker_stats_url}: {e}", extra={"event": "worker_stats_fetch_failed"}, ) # Fallback when worker is temporarily unreachable return { "pipeline": { "total_received": total_received, "total_processed": 0, "total_dlq": 0, "in_flight_buffer": 0, "last_flush_at": None, "worker_status": "unreachable", }, "breakdown": { "by_service_type": {}, "by_radio_access_type": {}, "dlq_by_reason": {}, }, }