feat(worker): complete Phase 3 - processing worker with normalization, micro-batching, and at-least-once offset management

app/worker/dlq.py

@@ -0,0 +1,66 @@
+import json
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+from app.api.producer import MessageProducer
+from app.common.config import config
+from app.common.logging import setup_logger
+from app.worker.metrics import worker_metrics
+
+logger = setup_logger("processing-worker")
+
+ALLOWED_FAILURE_REASONS = {
+    "missing_required_field",
+    "invalid_rat_value",
+    "invalid_epoch",
+    "malformed_json",
+    "normalization_error",
+    "ch_insert_failed",
+}
+
+
+def build_dlq_envelope(
+    original_payload: Any,
+    failure_reason: str,
+    failed_field: Optional[str] = None,
+    worker_instance: Optional[str] = None,
+) -> Dict[str, Any]:
+    """
+    Constructs a standardized DLQ envelope according to specification.
+    """
+    return {
+        "original_payload": original_payload,
+        "failure_reason": failure_reason,
+        "failed_field": failed_field,
+        "failed_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "worker_instance": worker_instance or config.worker.instance_id,
+    }
+
+
+def route_to_dlq(
+    producer: MessageProducer,
+    topic: str,
+    original_payload: Any,
+    failure_reason: str,
+    failed_field: Optional[str] = None,
+    raw_id: Optional[str] = None,
+) -> bool:
+    """
+    Wraps unprocessable record in a DLQ envelope, updates DLQ metric,
+    and publishes to the DLQ topic.
+    """
+    envelope = build_dlq_envelope(
+        original_payload=original_payload,
+        failure_reason=failure_reason,
+        failed_field=failed_field,
+    )
+    envelope_bytes = json.dumps(envelope, default=str).encode("utf-8")
+
+    success = producer.produce(topic=topic, key=raw_id, value=envelope_bytes)
+    worker_metrics.record_dlq(failure_reason, 1)
+
+    logger.warning(
+        f"Record routed to DLQ: reason={failure_reason}, field={failed_field}",
+        extra={"raw_id": raw_id, "event": "routed_to_dlq"},
+    )
+    return success