fix(worker): keep DLQ failures within the canonical taxonomy

app/worker/dlq.py

@@ -15,7 +15,6 @@ ALLOWED_FAILURE_REASONS = {
     "invalid_epoch",
     "malformed_json",
     "normalization_error",
-    "ch_insert_failed",
 }
 
 
@@ -58,13 +57,21 @@ def route_to_dlq(
     )
     envelope_bytes = json.dumps(envelope, default=str).encode("utf-8")
 
-    producer.produce(topic=topic, key=raw_id, value=envelope_bytes).result(
+    safe_key = None
+    if raw_id is not None:
+        try:
+            raw_id.encode("utf-8")
+            safe_key = raw_id
+        except UnicodeError:
+            safe_key = None
+
+    producer.produce(topic=topic, key=safe_key, value=envelope_bytes).result(
         timeout=config.kafka.delivery_timeout_seconds + 1
     )
     worker_metrics.record_dlq(failure_reason, 1)
 
     logger.warning(
         f"Record routed to DLQ: reason={failure_reason}, field={failed_field}",
-        extra={"raw_id": raw_id, "event": "routed_to_dlq"},
+        extra={"raw_id": safe_key, "event": "routed_to_dlq"},
     )
     return True

app/worker/normalizer.py

@@ -113,6 +113,11 @@ def normalize_cdr_payload(
         return None, "malformed_json", None, None
     raw_id = payload.get("raw_id")
     raw_id_str = raw_id if isinstance(raw_id, str) else None
+    if raw_id_str is not None:
+        try:
+            raw_id_str.encode("utf-8")
+        except UnicodeError:
+            raw_id_str = None
     try:
         validate_structure(payload)
     except PayloadError as exc:

tests/test_worker.py

@@ -262,10 +262,10 @@ class TestProcessingWorker(unittest.TestCase):
         msg = MockKafkaMessage(json.dumps(payload).encode("utf-8"))
         worker.process_single_message(msg)
 
-        # Record should have fallen back to DLQ with ch_insert_failed
+        # Record should have fallen back to DLQ with normalization_error
         self.assertEqual(len(self.dlq_producer.messages), 1)
         dlq_data = json.loads(self.dlq_producer.messages[0]["value"].decode("utf-8"))
-        self.assertEqual(dlq_data["failure_reason"], "ch_insert_failed")
+        self.assertEqual(dlq_data["failure_reason"], "normalization_error")
 
     def test_internal_stats_and_metrics_endpoints(self):
         # Verify /internal/stats