110 lines
3.9 KiB
Python
110 lines
3.9 KiB
Python
from datetime import datetime, timezone
|
|
from ipaddress import IPv4Address
|
|
from uuid import uuid4
|
|
|
|
from fastapi.testclient import TestClient
|
|
from prometheus_client import CONTENT_TYPE_LATEST
|
|
|
|
from core.domain.models import NormalizedCDR
|
|
from core.worker.observability.metrics import WorkerMetrics
|
|
from core.worker.observability.server import create_worker_observability_app
|
|
from core.worker.observability.stats import WorkerRuntimeStats
|
|
|
|
|
|
def create_sample_cdr() -> NormalizedCDR:
|
|
now = datetime.now(timezone.utc)
|
|
return NormalizedCDR(
|
|
cdr_id=uuid4(),
|
|
raw_id="raw-obs-1",
|
|
msisdn="+19875551234",
|
|
imsi="310410998877665",
|
|
service_type="4G_DATA",
|
|
radio_access_type="LTE_4G",
|
|
cell_id="4021_8812",
|
|
bytes_uploaded=100,
|
|
bytes_downloaded=200,
|
|
total_bytes=300,
|
|
duration_seconds=300,
|
|
session_start_time=now,
|
|
session_end_time=now,
|
|
source_ip=IPv4Address("10.240.12.1"),
|
|
)
|
|
|
|
|
|
def test_worker_observability_health_endpoint() -> None:
|
|
metrics = WorkerMetrics()
|
|
stats = WorkerRuntimeStats()
|
|
app = create_worker_observability_app(metrics=metrics, stats=stats)
|
|
client = TestClient(app)
|
|
|
|
response = client.get("/health")
|
|
assert response.status_code == 200
|
|
assert response.json() == {"status": "ok"}
|
|
|
|
|
|
def test_worker_observability_metrics_endpoint() -> None:
|
|
metrics = WorkerMetrics()
|
|
stats = WorkerRuntimeStats()
|
|
app = create_worker_observability_app(metrics=metrics, stats=stats)
|
|
client = TestClient(app)
|
|
|
|
response = client.get("/metrics")
|
|
assert response.status_code == 200
|
|
assert CONTENT_TYPE_LATEST in response.headers["content-type"]
|
|
assert "cdr_processed_total" in response.text
|
|
assert "cdr_dlq_total" in response.text
|
|
|
|
|
|
def test_worker_observability_internal_stats_endpoint() -> None:
|
|
metrics = WorkerMetrics()
|
|
stats = WorkerRuntimeStats()
|
|
app = create_worker_observability_app(metrics=metrics, stats=stats)
|
|
client = TestClient(app)
|
|
|
|
response = client.get("/internal/stats")
|
|
assert response.status_code == 200
|
|
body = response.json()
|
|
assert "pipeline" in body
|
|
assert "breakdown" in body
|
|
assert body["pipeline"]["total_processed"] == 0
|
|
assert body["pipeline"]["total_dlq"] == 0
|
|
|
|
|
|
def test_shared_instance_behavior_between_processing_and_observability() -> None:
|
|
# Processing and observability must share the identical metrics and stats instances
|
|
metrics = WorkerMetrics()
|
|
stats = WorkerRuntimeStats()
|
|
app = create_worker_observability_app(metrics=metrics, stats=stats)
|
|
client = TestClient(app)
|
|
|
|
# Simulate worker processing activity updating the instances
|
|
cdr = create_sample_cdr()
|
|
metrics.record_processed(cdr)
|
|
stats.record_processed(cdr)
|
|
|
|
metrics.record_dlq("invalid_rat_value")
|
|
stats.record_dlq("invalid_rat_value")
|
|
|
|
metrics.set_buffer_size(5)
|
|
stats.set_buffer_size(5)
|
|
|
|
# Verify that the observability server sees the exact same updates
|
|
stats_response = client.get("/internal/stats")
|
|
assert stats_response.status_code == 200
|
|
stats_body = stats_response.json()
|
|
|
|
assert stats_body["pipeline"]["total_processed"] == 1
|
|
assert stats_body["pipeline"]["total_dlq"] == 1
|
|
assert stats_body["pipeline"]["in_flight_buffer"] == 5
|
|
assert stats_body["breakdown"]["by_service_type"]["4G_DATA"] == 1
|
|
assert stats_body["breakdown"]["by_radio_access_type"]["LTE_4G"] == 1
|
|
assert stats_body["breakdown"]["dlq_by_reason"]["invalid_rat_value"] == 1
|
|
|
|
# Verify metrics endpoint output contains the recorded metrics
|
|
metrics_response = client.get("/metrics")
|
|
assert metrics_response.status_code == 200
|
|
assert 'cdr_processed_total{radio_access_type="LTE_4G",service_type="4G_DATA"} 1.0' in metrics_response.text
|
|
assert 'cdr_dlq_total{reason="invalid_rat_value"} 1.0' in metrics_response.text
|
|
assert 'cdr_in_flight_buffer_size 5.0' in metrics_response.text
|
|
|