feat(api): complete Phase 2 - framework-less asyncio Ingestion API with backpressure and metrics
This commit is contained in:
1
app/api/__init__.py
Normal file
1
app/api/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# app/api package
|
||||
37
app/api/metrics.py
Normal file
37
app/api/metrics.py
Normal file
@@ -0,0 +1,37 @@
|
||||
import threading
|
||||
|
||||
|
||||
class APIMetrics:
|
||||
"""
|
||||
In-memory metrics tracker for the Ingestion API.
|
||||
Thread-safe and async-safe counter for Prometheus scraping.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._received_total: int = 0
|
||||
|
||||
def increment_received(self, count: int = 1) -> None:
|
||||
with self._lock:
|
||||
self._received_total += count
|
||||
|
||||
@property
|
||||
def total_received(self) -> int:
|
||||
with self._lock:
|
||||
return self._received_total
|
||||
|
||||
def get_prometheus_metrics(self) -> str:
|
||||
"""
|
||||
Formats metrics into standard Prometheus text format.
|
||||
"""
|
||||
with self._lock:
|
||||
val = self._received_total
|
||||
|
||||
return (
|
||||
"# HELP cdr_api_received_total Total CDRs received by the API\n"
|
||||
"# TYPE cdr_api_received_total counter\n"
|
||||
f"cdr_api_received_total {val}\n"
|
||||
)
|
||||
|
||||
|
||||
api_metrics = APIMetrics()
|
||||
185
app/api/producer.py
Normal file
185
app/api/producer.py
Normal file
@@ -0,0 +1,185 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from app.common.config import config
|
||||
from app.common.logging import setup_logger
|
||||
|
||||
logger = setup_logger("ingestion-api")
|
||||
|
||||
|
||||
class MessageProducer(ABC):
|
||||
@abstractmethod
|
||||
def produce(self, topic: str, key: Optional[str], value: bytes) -> bool:
|
||||
"""Produce a message synchronously or enqueue in broker buffer."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def is_healthy(self) -> bool:
|
||||
"""Check if producer is healthy and connected to broker."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def flush(self, timeout: float = 5.0) -> int:
|
||||
"""Flush any pending messages."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Close producer connection."""
|
||||
pass
|
||||
|
||||
|
||||
class ConfluentKafkaProducer(MessageProducer):
|
||||
"""
|
||||
Production Kafka producer implementation wrapping confluent-kafka.
|
||||
Uses 'raw_id' as partition key and LZ4 compression.
|
||||
"""
|
||||
|
||||
def __init__(self, bootstrap_servers: Optional[str] = None):
|
||||
try:
|
||||
import confluent_kafka
|
||||
self._ck = confluent_kafka
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"confluent-kafka library is not installed. Run 'pip install confluent-kafka'."
|
||||
)
|
||||
|
||||
self.bootstrap_servers = bootstrap_servers or config.kafka.bootstrap_servers
|
||||
self._producer_config = {
|
||||
"bootstrap.servers": self.bootstrap_servers,
|
||||
"compression.type": "lz4",
|
||||
"queue.buffering.max.messages": 100000,
|
||||
"queue.buffering.max.ms": 5,
|
||||
"acks": "all",
|
||||
"retries": 3,
|
||||
}
|
||||
self._producer = self._ck.Producer(self._producer_config)
|
||||
self._healthy = True
|
||||
|
||||
def _delivery_report(self, err: Any, msg: Any) -> None:
|
||||
if err is not None:
|
||||
self._healthy = False
|
||||
raw_id = msg.key().decode("utf-8") if msg and msg.key() else None
|
||||
logger.error(
|
||||
f"Kafka delivery failed: {err}",
|
||||
extra={"raw_id": raw_id, "event": "kafka_produce_failed"},
|
||||
)
|
||||
else:
|
||||
self._healthy = True
|
||||
|
||||
def produce(self, topic: str, key: Optional[str], value: bytes) -> bool:
|
||||
try:
|
||||
key_bytes = key.encode("utf-8") if key else None
|
||||
self._producer.produce(
|
||||
topic=topic,
|
||||
key=key_bytes,
|
||||
value=value,
|
||||
on_delivery=self._delivery_report,
|
||||
)
|
||||
# Serve delivery callback queue non-blockingly
|
||||
self._producer.poll(0)
|
||||
return True
|
||||
except BufferError:
|
||||
logger.warning(
|
||||
"Kafka local producer buffer full",
|
||||
extra={"raw_id": key, "event": "kafka_buffer_full"},
|
||||
)
|
||||
self._producer.poll(0.1)
|
||||
return False
|
||||
except Exception as e:
|
||||
self._healthy = False
|
||||
logger.error(
|
||||
f"Kafka produce error: {e}",
|
||||
extra={"raw_id": key, "event": "kafka_error"},
|
||||
)
|
||||
return False
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
return self._healthy
|
||||
|
||||
def flush(self, timeout: float = 5.0) -> int:
|
||||
return self._producer.flush(timeout)
|
||||
|
||||
def close(self) -> None:
|
||||
self.flush()
|
||||
|
||||
|
||||
class MockMessageProducer(MessageProducer):
|
||||
"""
|
||||
In-memory mock producer for offline testing and verification.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.messages = []
|
||||
self._healthy = True
|
||||
|
||||
def produce(self, topic: str, key: Optional[str], value: bytes) -> bool:
|
||||
if not self._healthy:
|
||||
return False
|
||||
self.messages.append({"topic": topic, "key": key, "value": value})
|
||||
return True
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
return self._healthy
|
||||
|
||||
def set_healthy(self, healthy: bool) -> None:
|
||||
self._healthy = healthy
|
||||
|
||||
def flush(self, timeout: float = 5.0) -> int:
|
||||
return 0
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class AsyncProducerWorker:
|
||||
"""
|
||||
Background worker consuming from the bounded async queue and writing to the Kafka producer.
|
||||
"""
|
||||
|
||||
def __init__(self, queue: asyncio.Queue, producer: MessageProducer, topic: str):
|
||||
self.queue = queue
|
||||
self.producer = producer
|
||||
self.topic = topic
|
||||
self._running = False
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
|
||||
async def start(self) -> None:
|
||||
self._running = True
|
||||
self._task = asyncio.create_task(self._run_loop())
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while self._running:
|
||||
try:
|
||||
item = await self.queue.get()
|
||||
if item is None: # Sentinel to stop
|
||||
self.queue.task_done()
|
||||
break
|
||||
|
||||
key, value = item
|
||||
success = self.producer.produce(self.topic, key, value)
|
||||
if not success:
|
||||
logger.error(
|
||||
"Failed to publish message from async queue to Kafka",
|
||||
extra={"raw_id": key, "event": "queue_dispatch_failed"},
|
||||
)
|
||||
self.queue.task_done()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Unexpected error in async producer worker: {e}",
|
||||
extra={"event": "worker_unexpected_error"},
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._running = False
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self.producer.flush()
|
||||
287
app/api/server.py
Normal file
287
app/api/server.py
Normal file
@@ -0,0 +1,287 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from app.api.metrics import api_metrics
|
||||
from app.api.producer import (
|
||||
AsyncProducerWorker,
|
||||
ConfluentKafkaProducer,
|
||||
MessageProducer,
|
||||
MockMessageProducer,
|
||||
)
|
||||
from app.api.stats import get_aggregated_pipeline_stats
|
||||
from app.common.config import config
|
||||
from app.common.logging import setup_logger
|
||||
|
||||
logger = setup_logger("ingestion-api")
|
||||
|
||||
|
||||
class IngestionAPIServer:
|
||||
"""
|
||||
Framework-less HTTP/1.1 Ingestion API server built on standard library asyncio.
|
||||
Enforces bounded async queue backpressure and dispatches to Kafka.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: Optional[str] = None,
|
||||
port: Optional[int] = None,
|
||||
queue_max_size: Optional[int] = None,
|
||||
producer: Optional[MessageProducer] = None,
|
||||
worker_stats_url: Optional[str] = None,
|
||||
):
|
||||
self.host = host or config.api.host
|
||||
self.port = port or config.api.port
|
||||
self.queue_max_size = queue_max_size or config.api.queue_max_size
|
||||
self.worker_stats_url = worker_stats_url or config.api.worker_stats_url
|
||||
|
||||
self.queue: asyncio.Queue = asyncio.Queue(maxsize=self.queue_max_size)
|
||||
self.producer = producer or ConfluentKafkaProducer()
|
||||
self.dispatcher = AsyncProducerWorker(
|
||||
queue=self.queue,
|
||||
producer=self.producer,
|
||||
topic=config.kafka.raw_topic,
|
||||
)
|
||||
|
||||
self._server: Optional[asyncio.Server] = None
|
||||
self._running = False
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Start the async producer dispatcher and HTTP server."""
|
||||
await self.dispatcher.start()
|
||||
self._server = await asyncio.start_server(
|
||||
self._handle_client, self.host, self.port
|
||||
)
|
||||
self._running = True
|
||||
logger.info(
|
||||
f"Ingestion API listening on http://{self.host}:{self.port} (queue capacity: {self.queue_max_size})",
|
||||
extra={"event": "server_started"},
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the HTTP server and flush dispatcher queue."""
|
||||
self._running = False
|
||||
if self._server:
|
||||
self._server.close()
|
||||
await self._server.wait_closed()
|
||||
await self.dispatcher.stop()
|
||||
self.producer.close()
|
||||
logger.info("Ingestion API stopped cleanly", extra={"event": "server_stopped"})
|
||||
|
||||
async def _handle_client(
|
||||
self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
|
||||
) -> None:
|
||||
try:
|
||||
# Read request line
|
||||
request_line = await reader.readline()
|
||||
if not request_line:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
return
|
||||
|
||||
line_str = request_line.decode("utf-8", errors="replace").strip()
|
||||
parts = line_str.split(" ")
|
||||
if len(parts) < 2:
|
||||
await self._send_response(
|
||||
writer, 400, "application/json", b'{"error": "bad_request"}'
|
||||
)
|
||||
return
|
||||
|
||||
method = parts[0].upper()
|
||||
path = parts[1]
|
||||
|
||||
# Read headers
|
||||
headers = {}
|
||||
while True:
|
||||
header_line = await reader.readline()
|
||||
if not header_line or header_line == b"\r\n" or header_line == b"\n":
|
||||
break
|
||||
header_str = header_line.decode("utf-8", errors="replace").strip()
|
||||
if ":" in header_str:
|
||||
key, val = header_str.split(":", 1)
|
||||
headers[key.strip().lower()] = val.strip()
|
||||
|
||||
# Read body based on Content-Length
|
||||
content_length = int(headers.get("content-length", 0))
|
||||
body = b""
|
||||
if content_length > 0:
|
||||
body = await reader.readexactly(content_length)
|
||||
|
||||
# Route dispatch
|
||||
await self._dispatch_route(writer, method, path, headers, body)
|
||||
|
||||
except asyncio.IncompleteReadError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error handling HTTP request: {e}",
|
||||
extra={"event": "request_handler_error"},
|
||||
)
|
||||
try:
|
||||
err_body = json.dumps(
|
||||
{"error": "internal_error", "detail": str(e)}
|
||||
).encode("utf-8")
|
||||
await self._send_response(writer, 500, "application/json", err_body)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def _dispatch_route(
|
||||
self,
|
||||
writer: asyncio.StreamWriter,
|
||||
method: str,
|
||||
path: str,
|
||||
headers: dict,
|
||||
body: bytes,
|
||||
) -> None:
|
||||
# 1. Health check
|
||||
if method == "GET" and path in ("/health", "/healthz", "/live"):
|
||||
res = json.dumps(
|
||||
{"status": "ok", "queue_size": self.queue.qsize()}
|
||||
).encode("utf-8")
|
||||
await self._send_response(writer, 200, "application/json", res)
|
||||
return
|
||||
|
||||
# 2. Prometheus metrics
|
||||
if method == "GET" and path == "/metrics":
|
||||
res = api_metrics.get_prometheus_metrics().encode("utf-8")
|
||||
await self._send_response(
|
||||
writer, 200, "text/plain; version=0.0.4; charset=utf-8", res
|
||||
)
|
||||
return
|
||||
|
||||
# 3. Public Stats summary
|
||||
if method == "GET" and path == "/stats":
|
||||
stats = await get_aggregated_pipeline_stats(self.worker_stats_url)
|
||||
res = json.dumps(stats, indent=2).encode("utf-8")
|
||||
await self._send_response(writer, 200, "application/json", res)
|
||||
return
|
||||
|
||||
# 4. Raw CDR Ingestion endpoint
|
||||
if method == "POST" and path == "/api/v1/cdr/raw":
|
||||
await self._handle_raw_cdr_ingestion(writer, body)
|
||||
return
|
||||
|
||||
# 5. Not found
|
||||
err = json.dumps(
|
||||
{"error": "not_found", "detail": f"Cannot {method} {path}"}
|
||||
).encode("utf-8")
|
||||
await self._send_response(writer, 404, "application/json", err)
|
||||
|
||||
async def _handle_raw_cdr_ingestion(
|
||||
self, writer: asyncio.StreamWriter, body: bytes
|
||||
) -> None:
|
||||
# Check broker reachability
|
||||
if not self.producer.is_healthy():
|
||||
err = json.dumps(
|
||||
{
|
||||
"error": "broker_unavailable",
|
||||
"detail": "Kafka broker is unreachable",
|
||||
}
|
||||
).encode("utf-8")
|
||||
await self._send_response(writer, 503, "application/json", err)
|
||||
return
|
||||
|
||||
# Check queue capacity (Backpressure)
|
||||
if self.queue.full():
|
||||
err = json.dumps(
|
||||
{
|
||||
"error": "queue_full",
|
||||
"detail": "Internal async buffer is saturated (backpressure)",
|
||||
}
|
||||
).encode("utf-8")
|
||||
await self._send_response(writer, 429, "application/json", err)
|
||||
return
|
||||
|
||||
# Validate JSON format
|
||||
try:
|
||||
body_text = body.decode("utf-8")
|
||||
payload = json.loads(body_text)
|
||||
except json.JSONDecodeError as e:
|
||||
err = json.dumps(
|
||||
{
|
||||
"error": "invalid_json",
|
||||
"detail": f"Unexpected token at position {e.pos}: {e.msg}",
|
||||
}
|
||||
).encode("utf-8")
|
||||
await self._send_response(writer, 400, "application/json", err)
|
||||
return
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
err = json.dumps(
|
||||
{"error": "invalid_payload", "detail": "Expected a JSON object"}
|
||||
).encode("utf-8")
|
||||
await self._send_response(writer, 400, "application/json", err)
|
||||
return
|
||||
|
||||
raw_id = payload.get("raw_id")
|
||||
|
||||
# Put in bounded queue without blocking
|
||||
try:
|
||||
self.queue.put_nowait((raw_id, body))
|
||||
except asyncio.QueueFull:
|
||||
err = json.dumps(
|
||||
{
|
||||
"error": "queue_full",
|
||||
"detail": "Internal async buffer is saturated (backpressure)",
|
||||
}
|
||||
).encode("utf-8")
|
||||
await self._send_response(writer, 429, "application/json", err)
|
||||
return
|
||||
|
||||
# Increment metric
|
||||
api_metrics.increment_received(1)
|
||||
|
||||
# 202 Accepted
|
||||
res = json.dumps(
|
||||
{"status": "accepted", "raw_id": raw_id, "queued": True}
|
||||
).encode("utf-8")
|
||||
await self._send_response(writer, 202, "application/json", res)
|
||||
|
||||
async def _send_response(
|
||||
self,
|
||||
writer: asyncio.StreamWriter,
|
||||
status_code: int,
|
||||
content_type: str,
|
||||
body: bytes,
|
||||
) -> None:
|
||||
status_reasons = {
|
||||
200: "OK",
|
||||
202: "Accepted",
|
||||
400: "Bad Request",
|
||||
404: "Not Found",
|
||||
429: "Too Many Requests",
|
||||
500: "Internal Server Error",
|
||||
503: "Service Unavailable",
|
||||
}
|
||||
reason = status_reasons.get(status_code, "Unknown")
|
||||
headers = (
|
||||
f"HTTP/1.1 {status_code} {reason}\r\n"
|
||||
f"Content-Type: {content_type}\r\n"
|
||||
f"Content-Length: {len(body)}\r\n"
|
||||
f"Connection: close\r\n"
|
||||
f"\r\n"
|
||||
)
|
||||
writer.write(headers.encode("utf-8") + body)
|
||||
await writer.drain()
|
||||
|
||||
|
||||
async def main():
|
||||
server = IngestionAPIServer()
|
||||
await server.start()
|
||||
try:
|
||||
# Keep running until cancelled
|
||||
while True:
|
||||
await asyncio.sleep(3600)
|
||||
except (asyncio.CancelledError, KeyboardInterrupt):
|
||||
await server.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
72
app/api/stats.py
Normal file
72
app/api/stats.py
Normal file
@@ -0,0 +1,72 @@
|
||||
import asyncio
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from typing import Any, Dict
|
||||
|
||||
from app.api.metrics import api_metrics
|
||||
from app.common.logging import setup_logger
|
||||
|
||||
logger = setup_logger("ingestion-api")
|
||||
|
||||
|
||||
def _sync_fetch_worker_stats(url: str, timeout: float = 2.0) -> Dict[str, Any]:
|
||||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
if response.status == 200:
|
||||
content = response.read().decode("utf-8")
|
||||
return json.loads(content)
|
||||
raise ValueError(f"Worker returned status {response.status}")
|
||||
|
||||
|
||||
async def get_aggregated_pipeline_stats(worker_stats_url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Combines local Ingestion API statistics with remote runtime statistics
|
||||
from the Processing Worker.
|
||||
"""
|
||||
total_received = api_metrics.total_received
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
worker_stats = await loop.run_in_executor(
|
||||
None, _sync_fetch_worker_stats, worker_stats_url
|
||||
)
|
||||
|
||||
pipeline_data = worker_stats.get("pipeline", {})
|
||||
breakdown_data = worker_stats.get("breakdown", {})
|
||||
|
||||
return {
|
||||
"pipeline": {
|
||||
"total_received": total_received,
|
||||
"total_processed": pipeline_data.get("total_processed", 0),
|
||||
"total_dlq": pipeline_data.get("total_dlq", 0),
|
||||
"in_flight_buffer": pipeline_data.get("in_flight_buffer", 0),
|
||||
"last_flush_at": pipeline_data.get("last_flush_at"),
|
||||
},
|
||||
"breakdown": {
|
||||
"by_service_type": breakdown_data.get("by_service_type", {}),
|
||||
"by_radio_access_type": breakdown_data.get("by_radio_access_type", {}),
|
||||
"dlq_by_reason": breakdown_data.get("dlq_by_reason", {}),
|
||||
},
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Unable to fetch worker stats from {worker_stats_url}: {e}",
|
||||
extra={"event": "worker_stats_fetch_failed"},
|
||||
)
|
||||
# Fallback when worker is temporarily unreachable
|
||||
return {
|
||||
"pipeline": {
|
||||
"total_received": total_received,
|
||||
"total_processed": 0,
|
||||
"total_dlq": 0,
|
||||
"in_flight_buffer": 0,
|
||||
"last_flush_at": None,
|
||||
"worker_status": "unreachable",
|
||||
},
|
||||
"breakdown": {
|
||||
"by_service_type": {},
|
||||
"by_radio_access_type": {},
|
||||
"dlq_by_reason": {},
|
||||
},
|
||||
}
|
||||
Reference in New Issue
Block a user