from datetime import datetime, timezone from ipaddress import IPv4Address from uuid import uuid4 from fastapi.testclient import TestClient from prometheus_client import CONTENT_TYPE_LATEST from core.domain.models import NormalizedCDR from core.worker.observability.metrics import WorkerMetrics from core.worker.observability.server import create_worker_observability_app from core.worker.observability.stats import WorkerRuntimeStats def create_sample_cdr() -> NormalizedCDR: now = datetime.now(timezone.utc) return NormalizedCDR( cdr_id=uuid4(), raw_id="raw-obs-1", msisdn="+19875551234", imsi="310410998877665", service_type="4G_DATA", radio_access_type="LTE_4G", cell_id="4021_8812", bytes_uploaded=100, bytes_downloaded=200, total_bytes=300, duration_seconds=300, session_start_time=now, session_end_time=now, source_ip=IPv4Address("10.240.12.1"), ) def test_worker_observability_health_endpoint() -> None: metrics = WorkerMetrics() stats = WorkerRuntimeStats() app = create_worker_observability_app(metrics=metrics, stats=stats) client = TestClient(app) response = client.get("/health") assert response.status_code == 200 assert response.json() == {"status": "ok"} def test_worker_observability_metrics_endpoint() -> None: metrics = WorkerMetrics() stats = WorkerRuntimeStats() app = create_worker_observability_app(metrics=metrics, stats=stats) client = TestClient(app) response = client.get("/metrics") assert response.status_code == 200 assert CONTENT_TYPE_LATEST in response.headers["content-type"] assert "cdr_processed_total" in response.text assert "cdr_dlq_total" in response.text def test_worker_observability_internal_stats_endpoint() -> None: metrics = WorkerMetrics() stats = WorkerRuntimeStats() app = create_worker_observability_app(metrics=metrics, stats=stats) client = TestClient(app) response = client.get("/internal/stats") assert response.status_code == 200 body = response.json() assert "pipeline" in body assert "breakdown" in body assert body["pipeline"]["total_processed"] == 0 assert body["pipeline"]["total_dlq"] == 0 def test_shared_instance_behavior_between_processing_and_observability() -> None: # Processing and observability must share the identical metrics and stats instances metrics = WorkerMetrics() stats = WorkerRuntimeStats() app = create_worker_observability_app(metrics=metrics, stats=stats) client = TestClient(app) # Simulate worker processing activity updating the instances cdr = create_sample_cdr() metrics.record_processed(cdr) stats.record_processed(cdr) metrics.record_dlq("invalid_rat_value") stats.record_dlq("invalid_rat_value") metrics.set_buffer_size(5) stats.set_buffer_size(5) # Verify that the observability server sees the exact same updates stats_response = client.get("/internal/stats") assert stats_response.status_code == 200 stats_body = stats_response.json() assert stats_body["pipeline"]["total_processed"] == 1 assert stats_body["pipeline"]["total_dlq"] == 1 assert stats_body["pipeline"]["in_flight_buffer"] == 5 assert stats_body["breakdown"]["by_service_type"]["4G_DATA"] == 1 assert stats_body["breakdown"]["by_radio_access_type"]["LTE_4G"] == 1 assert stats_body["breakdown"]["dlq_by_reason"]["invalid_rat_value"] == 1 # Verify metrics endpoint output contains the recorded metrics metrics_response = client.get("/metrics") assert metrics_response.status_code == 200 assert 'cdr_processed_total{radio_access_type="LTE_4G",service_type="4G_DATA"} 1.0' in metrics_response.text assert 'cdr_dlq_total{reason="invalid_rat_value"} 1.0' in metrics_response.text assert 'cdr_in_flight_buffer_size 5.0' in metrics_response.text