telecom-cdr-pipeline-ai/app/worker/metrics.py

import threading
from collections import defaultdict
from typing import Any, Dict, List, Optional


def escape_label(value: str) -> str:
    return value.replace(chr(92), chr(92)*2).replace('"', chr(92)+'"').replace(chr(10), chr(92)+'n')


class SimpleHistogram:
    """
    Lightweight, thread-safe Prometheus-compatible histogram.
    """

    def __init__(self, name: str, help_text: str, buckets: List[float]):
        self.name = name
        self.help_text = help_text
        self.buckets = sorted(buckets)
        self._counts: Dict[float, int] = {b: 0 for b in self.buckets}
        self._count = 0
        self._sum = 0.0
        self._lock = threading.Lock()

    def observe(self, value: float) -> None:
        with self._lock:
            self._count += 1
            self._sum += value
            for b in self.buckets:
                if value <= b:
                    self._counts[b] += 1

    def to_prometheus(self) -> str:
        lines = [
            f"# HELP {self.name} {self.help_text}",
            f"# TYPE {self.name} histogram",
        ]
        with self._lock:
            for b in self.buckets:
                lines.append(f'{self.name}_bucket{{le="{b}"}} {self._counts[b]}')
            lines.append(f'{self.name}_bucket{{le="+Inf"}} {self._count}')
            lines.append(f"{self.name}_count {self._count}")
            lines.append(f"{self.name}_sum {self._sum:.4f}")
        return "\n".join(lines) + "\n"


class WorkerMetrics:
    """
    Thread-safe operational metrics collector for the Processing Worker.
    Provides data for /metrics (Prometheus) and /internal/stats (Ingestion API aggregator).
    """

    def __init__(self):
        self._lock = threading.Lock()
        self._processed_total: Dict[tuple, int] = defaultdict(int)
        self._dlq_total: Dict[str, int] = defaultdict(int)
        self._in_flight_buffer_size: int = 0
        self._last_flush_at: Optional[str] = None
        self._consumer_lag: Dict[str, int] = defaultdict(int)

        # Histograms
        self.kafka_to_ch_latency = SimpleHistogram(
            name="cdr_kafka_to_clickhouse_latency_seconds",
            help_text="End-to-end latency from Kafka produce to ClickHouse write",
            buckets=[0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0],
        )
        self.batch_flush_duration = SimpleHistogram(
            name="cdr_batch_flush_duration_seconds",
            help_text="ClickHouse batch flush duration in seconds",
            buckets=[0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0],
        )
        self.batch_size_at_flush = SimpleHistogram(
            name="cdr_batch_size_at_flush",
            help_text="Batch sizes at flush time",
            buckets=[10, 50, 100, 250, 500, 1000],
        )

    def record_processed(
        self, service_type: str, radio_access_type: str, count: int = 1
    ) -> None:
        with self._lock:
            self._processed_total[(service_type, radio_access_type)] += count

    def record_dlq(self, reason: str, count: int = 1) -> None:
        with self._lock:
            self._dlq_total[reason] += count

    def set_in_flight_buffer(self, size: int) -> None:
        with self._lock:
            self._in_flight_buffer_size = size

    def set_last_flush_at(self, timestamp_iso: str) -> None:
        with self._lock:
            self._last_flush_at = timestamp_iso

    def set_consumer_lag(self, partition: int, lag: int) -> None:
        with self._lock:
            self._consumer_lag[str(partition)] = lag

    def get_internal_stats(self) -> Dict[str, Any]:
        """
        Returns structured dictionary for GET /internal/stats.
        """
        with self._lock:
            total_processed = sum(self._processed_total.values())
            total_dlq = sum(self._dlq_total.values())
            in_flight = self._in_flight_buffer_size
            last_flush = self._last_flush_at

            # Breakdowns
            by_service: Dict[str, int] = defaultdict(int)
            by_rat: Dict[str, int] = defaultdict(int)
            for (svc, rat), cnt in self._processed_total.items():
                by_service[svc] += cnt
                by_rat[rat] += cnt

            dlq_reasons = dict(self._dlq_total)

        return {
            "pipeline": {
                "total_processed": total_processed,
                "total_dlq": total_dlq,
                "in_flight_buffer": in_flight,
                "last_flush_at": last_flush,
            },
            "breakdown": {
                "by_service_type": dict(by_service),
                "by_radio_access_type": dict(by_rat),
                "dlq_by_reason": dlq_reasons,
            },
        }

    def get_prometheus_metrics(self) -> str:
        """
        Formats all worker metrics into standard Prometheus exposition format.
        """
        lines = []

        # 1. cdr_processed_total
        lines.append("# HELP cdr_processed_total Total CDRs successfully written to ClickHouse")
        lines.append("# TYPE cdr_processed_total counter")
        with self._lock:
            for (svc, rat), count in self._processed_total.items():
                lines.append(f'cdr_processed_total{{service_type="{escape_label(svc)}",radio_access_type="{escape_label(rat)}"}} {count}')

            # 2. cdr_dlq_total
            lines.append("# HELP cdr_dlq_total Total CDRs routed to DLQ by reason")
            lines.append("# TYPE cdr_dlq_total counter")
            for reason, count in self._dlq_total.items():
                lines.append(f'cdr_dlq_total{{reason="{escape_label(reason)}"}} {count}')

            # 3. Gauges
            lines.append("# HELP cdr_in_flight_buffer_size Current in-memory buffer size")
            lines.append("# TYPE cdr_in_flight_buffer_size gauge")
            lines.append(f"cdr_in_flight_buffer_size {self._in_flight_buffer_size}")

            lines.append("# HELP cdr_kafka_consumer_lag Kafka consumer lag per partition")
            lines.append("# TYPE cdr_kafka_consumer_lag gauge")
            for part, lag in self._consumer_lag.items():
                lines.append(f'cdr_kafka_consumer_lag{{partition="{part}"}} {lag}')

        # 4. Histograms
        lines.append(self.kafka_to_ch_latency.to_prometheus().strip())
        lines.append(self.batch_flush_duration.to_prometheus().strip())
        lines.append(self.batch_size_at_flush.to_prometheus().strip())

        return "\n".join(lines) + "\n"


worker_metrics = WorkerMetrics()