import asyncio import json import logging from typing import Optional, Tuple from app.api.metrics import api_metrics from app.api.producer import ( AsyncProducerWorker, ConfluentKafkaProducer, MessageProducer, MockMessageProducer, ) from app.api.stats import get_aggregated_pipeline_stats from app.common.config import config from app.common.logging import setup_logger logger = setup_logger("ingestion-api") class IngestionAPIServer: """ Framework-less HTTP/1.1 Ingestion API server built on standard library asyncio. Enforces bounded async queue backpressure and dispatches to Kafka. """ def __init__( self, host: Optional[str] = None, port: Optional[int] = None, queue_max_size: Optional[int] = None, producer: Optional[MessageProducer] = None, worker_stats_url: Optional[str] = None, ): self.host = host or config.api.host self.port = port or config.api.port self.queue_max_size = queue_max_size or config.api.queue_max_size self.worker_stats_url = worker_stats_url or config.api.worker_stats_url self.queue: asyncio.Queue = asyncio.Queue(maxsize=self.queue_max_size) self.producer = producer or ConfluentKafkaProducer() self.dispatcher = AsyncProducerWorker( queue=self.queue, producer=self.producer, topic=config.kafka.raw_topic, ) self._server: Optional[asyncio.Server] = None self._running = False async def start(self) -> None: """Start the async producer dispatcher and HTTP server.""" await self.dispatcher.start() self._server = await asyncio.start_server( self._handle_client, self.host, self.port ) self._running = True logger.info( f"Ingestion API listening on http://{self.host}:{self.port} (queue capacity: {self.queue_max_size})", extra={"event": "server_started"}, ) async def stop(self) -> None: """Stop the HTTP server and flush dispatcher queue.""" self._running = False if self._server: self._server.close() await self._server.wait_closed() await self.dispatcher.stop() self.producer.close() logger.info("Ingestion API stopped cleanly", extra={"event": "server_stopped"}) async def _handle_client( self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter ) -> None: try: # Read request line request_line = await reader.readline() if not request_line: writer.close() await writer.wait_closed() return line_str = request_line.decode("utf-8", errors="replace").strip() parts = line_str.split(" ") if len(parts) < 2: await self._send_response( writer, 400, "application/json", b'{"error": "bad_request"}' ) return method = parts[0].upper() path = parts[1] # Read headers headers = {} while True: header_line = await reader.readline() if not header_line or header_line == b"\r\n" or header_line == b"\n": break header_str = header_line.decode("utf-8", errors="replace").strip() if ":" in header_str: key, val = header_str.split(":", 1) headers[key.strip().lower()] = val.strip() # Read body based on Content-Length content_length = int(headers.get("content-length", 0)) body = b"" if content_length > 0: body = await reader.readexactly(content_length) # Route dispatch await self._dispatch_route(writer, method, path, headers, body) except asyncio.IncompleteReadError: pass except Exception as e: logger.error( f"Error handling HTTP request: {e}", extra={"event": "request_handler_error"}, ) try: err_body = json.dumps( {"error": "internal_error", "detail": str(e)} ).encode("utf-8") await self._send_response(writer, 500, "application/json", err_body) except Exception: pass finally: try: writer.close() await writer.wait_closed() except Exception: pass async def _dispatch_route( self, writer: asyncio.StreamWriter, method: str, path: str, headers: dict, body: bytes, ) -> None: # 1. Health check if method == "GET" and path in ("/health", "/healthz", "/live"): res = json.dumps( {"status": "ok", "queue_size": self.queue.qsize()} ).encode("utf-8") await self._send_response(writer, 200, "application/json", res) return # 2. Prometheus metrics if method == "GET" and path == "/metrics": res = api_metrics.get_prometheus_metrics().encode("utf-8") await self._send_response( writer, 200, "text/plain; version=0.0.4; charset=utf-8", res ) return # 3. Public Stats summary if method == "GET" and path == "/stats": stats = await get_aggregated_pipeline_stats(self.worker_stats_url) res = json.dumps(stats, indent=2).encode("utf-8") await self._send_response(writer, 200, "application/json", res) return # 4. Raw CDR Ingestion endpoint if method == "POST" and path == "/api/v1/cdr/raw": await self._handle_raw_cdr_ingestion(writer, body) return # 5. Not found err = json.dumps( {"error": "not_found", "detail": f"Cannot {method} {path}"} ).encode("utf-8") await self._send_response(writer, 404, "application/json", err) async def _handle_raw_cdr_ingestion( self, writer: asyncio.StreamWriter, body: bytes ) -> None: # Check broker reachability if not self.producer.is_healthy(): err = json.dumps( { "error": "broker_unavailable", "detail": "Kafka broker is unreachable", } ).encode("utf-8") await self._send_response(writer, 503, "application/json", err) return # Check queue capacity (Backpressure) if self.queue.full(): err = json.dumps( { "error": "queue_full", "detail": "Internal async buffer is saturated (backpressure)", } ).encode("utf-8") await self._send_response(writer, 429, "application/json", err) return # Validate JSON format try: body_text = body.decode("utf-8") payload = json.loads(body_text) except json.JSONDecodeError as e: err = json.dumps( { "error": "invalid_json", "detail": f"Unexpected token at position {e.pos}: {e.msg}", } ).encode("utf-8") await self._send_response(writer, 400, "application/json", err) return if not isinstance(payload, dict): err = json.dumps( {"error": "invalid_payload", "detail": "Expected a JSON object"} ).encode("utf-8") await self._send_response(writer, 400, "application/json", err) return raw_id = payload.get("raw_id") # Put in bounded queue without blocking try: self.queue.put_nowait((raw_id, body)) except asyncio.QueueFull: err = json.dumps( { "error": "queue_full", "detail": "Internal async buffer is saturated (backpressure)", } ).encode("utf-8") await self._send_response(writer, 429, "application/json", err) return # Increment metric api_metrics.increment_received(1) # 202 Accepted res = json.dumps( {"status": "accepted", "raw_id": raw_id, "queued": True} ).encode("utf-8") await self._send_response(writer, 202, "application/json", res) async def _send_response( self, writer: asyncio.StreamWriter, status_code: int, content_type: str, body: bytes, ) -> None: status_reasons = { 200: "OK", 202: "Accepted", 400: "Bad Request", 404: "Not Found", 429: "Too Many Requests", 500: "Internal Server Error", 503: "Service Unavailable", } reason = status_reasons.get(status_code, "Unknown") headers = ( f"HTTP/1.1 {status_code} {reason}\r\n" f"Content-Type: {content_type}\r\n" f"Content-Length: {len(body)}\r\n" f"Connection: close\r\n" f"\r\n" ) writer.write(headers.encode("utf-8") + body) await writer.drain() async def main(): server = IngestionAPIServer() await server.start() try: # Keep running until cancelled while True: await asyncio.sleep(3600) except (asyncio.CancelledError, KeyboardInterrupt): await server.stop() if __name__ == "__main__": asyncio.run(main())