288 lines
9.5 KiB
Python
288 lines
9.5 KiB
Python
import asyncio
|
|
import json
|
|
import logging
|
|
from typing import Optional, Tuple
|
|
|
|
from app.api.metrics import api_metrics
|
|
from app.api.producer import (
|
|
AsyncProducerWorker,
|
|
ConfluentKafkaProducer,
|
|
MessageProducer,
|
|
MockMessageProducer,
|
|
)
|
|
from app.api.stats import get_aggregated_pipeline_stats
|
|
from app.common.config import config
|
|
from app.common.logging import setup_logger
|
|
|
|
logger = setup_logger("ingestion-api")
|
|
|
|
|
|
class IngestionAPIServer:
|
|
"""
|
|
Framework-less HTTP/1.1 Ingestion API server built on standard library asyncio.
|
|
Enforces bounded async queue backpressure and dispatches to Kafka.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
host: Optional[str] = None,
|
|
port: Optional[int] = None,
|
|
queue_max_size: Optional[int] = None,
|
|
producer: Optional[MessageProducer] = None,
|
|
worker_stats_url: Optional[str] = None,
|
|
):
|
|
self.host = host or config.api.host
|
|
self.port = port or config.api.port
|
|
self.queue_max_size = queue_max_size or config.api.queue_max_size
|
|
self.worker_stats_url = worker_stats_url or config.api.worker_stats_url
|
|
|
|
self.queue: asyncio.Queue = asyncio.Queue(maxsize=self.queue_max_size)
|
|
self.producer = producer or ConfluentKafkaProducer()
|
|
self.dispatcher = AsyncProducerWorker(
|
|
queue=self.queue,
|
|
producer=self.producer,
|
|
topic=config.kafka.raw_topic,
|
|
)
|
|
|
|
self._server: Optional[asyncio.Server] = None
|
|
self._running = False
|
|
|
|
async def start(self) -> None:
|
|
"""Start the async producer dispatcher and HTTP server."""
|
|
await self.dispatcher.start()
|
|
self._server = await asyncio.start_server(
|
|
self._handle_client, self.host, self.port
|
|
)
|
|
self._running = True
|
|
logger.info(
|
|
f"Ingestion API listening on http://{self.host}:{self.port} (queue capacity: {self.queue_max_size})",
|
|
extra={"event": "server_started"},
|
|
)
|
|
|
|
async def stop(self) -> None:
|
|
"""Stop the HTTP server and flush dispatcher queue."""
|
|
self._running = False
|
|
if self._server:
|
|
self._server.close()
|
|
await self._server.wait_closed()
|
|
await self.dispatcher.stop()
|
|
self.producer.close()
|
|
logger.info("Ingestion API stopped cleanly", extra={"event": "server_stopped"})
|
|
|
|
async def _handle_client(
|
|
self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
|
|
) -> None:
|
|
try:
|
|
# Read request line
|
|
request_line = await reader.readline()
|
|
if not request_line:
|
|
writer.close()
|
|
await writer.wait_closed()
|
|
return
|
|
|
|
line_str = request_line.decode("utf-8", errors="replace").strip()
|
|
parts = line_str.split(" ")
|
|
if len(parts) < 2:
|
|
await self._send_response(
|
|
writer, 400, "application/json", b'{"error": "bad_request"}'
|
|
)
|
|
return
|
|
|
|
method = parts[0].upper()
|
|
path = parts[1]
|
|
|
|
# Read headers
|
|
headers = {}
|
|
while True:
|
|
header_line = await reader.readline()
|
|
if not header_line or header_line == b"\r\n" or header_line == b"\n":
|
|
break
|
|
header_str = header_line.decode("utf-8", errors="replace").strip()
|
|
if ":" in header_str:
|
|
key, val = header_str.split(":", 1)
|
|
headers[key.strip().lower()] = val.strip()
|
|
|
|
# Read body based on Content-Length
|
|
content_length = int(headers.get("content-length", 0))
|
|
body = b""
|
|
if content_length > 0:
|
|
body = await reader.readexactly(content_length)
|
|
|
|
# Route dispatch
|
|
await self._dispatch_route(writer, method, path, headers, body)
|
|
|
|
except asyncio.IncompleteReadError:
|
|
pass
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Error handling HTTP request: {e}",
|
|
extra={"event": "request_handler_error"},
|
|
)
|
|
try:
|
|
err_body = json.dumps(
|
|
{"error": "internal_error", "detail": str(e)}
|
|
).encode("utf-8")
|
|
await self._send_response(writer, 500, "application/json", err_body)
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
try:
|
|
writer.close()
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
|
|
async def _dispatch_route(
|
|
self,
|
|
writer: asyncio.StreamWriter,
|
|
method: str,
|
|
path: str,
|
|
headers: dict,
|
|
body: bytes,
|
|
) -> None:
|
|
# 1. Health check
|
|
if method == "GET" and path in ("/health", "/healthz", "/live"):
|
|
res = json.dumps(
|
|
{"status": "ok", "queue_size": self.queue.qsize()}
|
|
).encode("utf-8")
|
|
await self._send_response(writer, 200, "application/json", res)
|
|
return
|
|
|
|
# 2. Prometheus metrics
|
|
if method == "GET" and path == "/metrics":
|
|
res = api_metrics.get_prometheus_metrics().encode("utf-8")
|
|
await self._send_response(
|
|
writer, 200, "text/plain; version=0.0.4; charset=utf-8", res
|
|
)
|
|
return
|
|
|
|
# 3. Public Stats summary
|
|
if method == "GET" and path == "/stats":
|
|
stats = await get_aggregated_pipeline_stats(self.worker_stats_url)
|
|
res = json.dumps(stats, indent=2).encode("utf-8")
|
|
await self._send_response(writer, 200, "application/json", res)
|
|
return
|
|
|
|
# 4. Raw CDR Ingestion endpoint
|
|
if method == "POST" and path == "/api/v1/cdr/raw":
|
|
await self._handle_raw_cdr_ingestion(writer, body)
|
|
return
|
|
|
|
# 5. Not found
|
|
err = json.dumps(
|
|
{"error": "not_found", "detail": f"Cannot {method} {path}"}
|
|
).encode("utf-8")
|
|
await self._send_response(writer, 404, "application/json", err)
|
|
|
|
async def _handle_raw_cdr_ingestion(
|
|
self, writer: asyncio.StreamWriter, body: bytes
|
|
) -> None:
|
|
# Check broker reachability
|
|
if not self.producer.is_healthy():
|
|
err = json.dumps(
|
|
{
|
|
"error": "broker_unavailable",
|
|
"detail": "Kafka broker is unreachable",
|
|
}
|
|
).encode("utf-8")
|
|
await self._send_response(writer, 503, "application/json", err)
|
|
return
|
|
|
|
# Check queue capacity (Backpressure)
|
|
if self.queue.full():
|
|
err = json.dumps(
|
|
{
|
|
"error": "queue_full",
|
|
"detail": "Internal async buffer is saturated (backpressure)",
|
|
}
|
|
).encode("utf-8")
|
|
await self._send_response(writer, 429, "application/json", err)
|
|
return
|
|
|
|
# Validate JSON format
|
|
try:
|
|
body_text = body.decode("utf-8")
|
|
payload = json.loads(body_text)
|
|
except json.JSONDecodeError as e:
|
|
err = json.dumps(
|
|
{
|
|
"error": "invalid_json",
|
|
"detail": f"Unexpected token at position {e.pos}: {e.msg}",
|
|
}
|
|
).encode("utf-8")
|
|
await self._send_response(writer, 400, "application/json", err)
|
|
return
|
|
|
|
if not isinstance(payload, dict):
|
|
err = json.dumps(
|
|
{"error": "invalid_payload", "detail": "Expected a JSON object"}
|
|
).encode("utf-8")
|
|
await self._send_response(writer, 400, "application/json", err)
|
|
return
|
|
|
|
raw_id = payload.get("raw_id")
|
|
|
|
# Put in bounded queue without blocking
|
|
try:
|
|
self.queue.put_nowait((raw_id, body))
|
|
except asyncio.QueueFull:
|
|
err = json.dumps(
|
|
{
|
|
"error": "queue_full",
|
|
"detail": "Internal async buffer is saturated (backpressure)",
|
|
}
|
|
).encode("utf-8")
|
|
await self._send_response(writer, 429, "application/json", err)
|
|
return
|
|
|
|
# Increment metric
|
|
api_metrics.increment_received(1)
|
|
|
|
# 202 Accepted
|
|
res = json.dumps(
|
|
{"status": "accepted", "raw_id": raw_id, "queued": True}
|
|
).encode("utf-8")
|
|
await self._send_response(writer, 202, "application/json", res)
|
|
|
|
async def _send_response(
|
|
self,
|
|
writer: asyncio.StreamWriter,
|
|
status_code: int,
|
|
content_type: str,
|
|
body: bytes,
|
|
) -> None:
|
|
status_reasons = {
|
|
200: "OK",
|
|
202: "Accepted",
|
|
400: "Bad Request",
|
|
404: "Not Found",
|
|
429: "Too Many Requests",
|
|
500: "Internal Server Error",
|
|
503: "Service Unavailable",
|
|
}
|
|
reason = status_reasons.get(status_code, "Unknown")
|
|
headers = (
|
|
f"HTTP/1.1 {status_code} {reason}\r\n"
|
|
f"Content-Type: {content_type}\r\n"
|
|
f"Content-Length: {len(body)}\r\n"
|
|
f"Connection: close\r\n"
|
|
f"\r\n"
|
|
)
|
|
writer.write(headers.encode("utf-8") + body)
|
|
await writer.drain()
|
|
|
|
|
|
async def main():
|
|
server = IngestionAPIServer()
|
|
await server.start()
|
|
try:
|
|
# Keep running until cancelled
|
|
while True:
|
|
await asyncio.sleep(3600)
|
|
except (asyncio.CancelledError, KeyboardInterrupt):
|
|
await server.stop()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|