feat: add bounded kafka ingestion pipeline

tests/integration/test_ingestion_api.py

@@ -1,9 +1,58 @@
+from collections.abc import Mapping
+from typing import Any
+
+import pytest
 from fastapi.testclient import TestClient
 
-from core.api.main import app
+from core.api.main import create_app
+from core.api.publisher import (
+    IngestionQueueFull,
+    KafkaPublishError,
+)
 
 
-client = TestClient(app)
+class FakePublisher:
+    def __init__(self) -> None:
+        self.messages: list[
+            tuple[str, Mapping[str, Any]]
+        ] = []
+
+        self.failure: Exception | None = None
+
+    async def start(self) -> None:
+        pass
+
+    async def stop(self) -> None:
+        pass
+
+    async def publish(
+        self,
+        payload: Mapping[str, Any],
+        key: str,
+    ) -> None:
+        if self.failure is not None:
+            raise self.failure
+
+        self.messages.append(
+            (key, payload)
+        )
+
+
+@pytest.fixture
+def publisher() -> FakePublisher:
+    return FakePublisher()
+
+
+@pytest.fixture
+def client(
+    publisher: FakePublisher,
+):
+    app = create_app(
+        publisher=publisher,
+    )
+
+    with TestClient(app) as test_client:
+        yield test_client
 
 
 def valid_payload() -> dict:
@@ -23,7 +72,7 @@ def valid_payload() -> dict:
     }
 
 
-def test_health_endpoint() -> None:
+def test_health_endpoint(client: TestClient) -> None:
     response = client.get("/health")
 
     assert response.status_code == 200
@@ -32,7 +81,10 @@ def test_health_endpoint() -> None:
     }
 
 
-def test_valid_cdr_returns_202() -> None:
+def test_valid_cdr_returns_202(
+    client: TestClient,
+    publisher: FakePublisher,
+) -> None:
     payload = valid_payload()
 
     response = client.post(
@@ -47,8 +99,15 @@ def test_valid_cdr_returns_202() -> None:
         "raw_id": "raw-test-001",
     }
 
+    assert len(publisher.messages) == 1
 
-def test_missing_required_field_returns_400() -> None:
+    key, published_payload = publisher.messages[0]
+
+    assert key == "raw-test-001"
+    assert published_payload == payload
+
+
+def test_missing_required_field_returns_400(client: TestClient) -> None:
     payload = valid_payload()
     del payload["caller_num"]
 
@@ -65,7 +124,7 @@ def test_missing_required_field_returns_400() -> None:
     assert body["failed_field"] == "caller_num"
 
 
-def test_invalid_rat_returns_400() -> None:
+def test_invalid_rat_returns_400(client: TestClient) -> None:
     payload = valid_payload()
     payload["rat"] = 99
 
@@ -82,7 +141,7 @@ def test_invalid_rat_returns_400() -> None:
     assert body["failed_field"] == "rat"
 
 
-def test_invalid_epoch_returns_400() -> None:
+def test_invalid_epoch_returns_400(client: TestClient) -> None:
     payload = valid_payload()
 
     payload["epoch_start"] = 200
@@ -99,7 +158,8 @@ def test_invalid_epoch_returns_400() -> None:
 
     assert body["error"] == "invalid_epoch"
 
-def test_malformed_json_returns_400() -> None:
+
+def test_malformed_json_returns_400(client: TestClient) -> None:
     response = client.post(
         "/api/v1/cdr/raw",
         content='{"raw_id": invalid}',
@@ -114,7 +174,8 @@ def test_malformed_json_returns_400() -> None:
 
     assert body["error"] == "invalid_json"
 
-def test_non_object_json_returns_400() -> None:
+
+def test_non_object_json_returns_400(client: TestClient) -> None:
     response = client.post(
         "/api/v1/cdr/raw",
         json=[
@@ -129,3 +190,39 @@ def test_non_object_json_returns_400() -> None:
     body = response.json()
 
     assert body["error"] == "invalid_payload"
+
+
+def test_queue_full_returns_429(
+    client: TestClient,
+    publisher: FakePublisher,
+) -> None:
+    publisher.failure = IngestionQueueFull(
+        "Ingestion queue is full."
+    )
+
+    response = client.post(
+        "/api/v1/cdr/raw",
+        json=valid_payload(),
+    )
+
+    assert response.status_code == 429
+
+    assert response.json()["error"] == "backpressure"
+
+
+def test_kafka_failure_returns_503(
+    client: TestClient,
+    publisher: FakePublisher,
+) -> None:
+    publisher.failure = KafkaPublishError(
+        "Kafka unavailable."
+    )
+
+    response = client.post(
+        "/api/v1/cdr/raw",
+        json=valid_payload(),
+    )
+
+    assert response.status_code == 503
+
+    assert response.json()["error"] == "kafka_unavailable"