fix(worker): keep DLQ failures within the canonical taxonomy

app/worker/dlq.py

@@ -15,7 +15,6 @@ ALLOWED_FAILURE_REASONS = {
     "invalid_epoch",
     "malformed_json",
     "normalization_error",
-    "ch_insert_failed",
 }
 
 
@@ -58,13 +57,21 @@ def route_to_dlq(
     )
     envelope_bytes = json.dumps(envelope, default=str).encode("utf-8")
 
-    producer.produce(topic=topic, key=raw_id, value=envelope_bytes).result(
+    safe_key = None
+    if raw_id is not None:
+        try:
+            raw_id.encode("utf-8")
+            safe_key = raw_id
+        except UnicodeError:
+            safe_key = None
+
+    producer.produce(topic=topic, key=safe_key, value=envelope_bytes).result(
         timeout=config.kafka.delivery_timeout_seconds + 1
     )
     worker_metrics.record_dlq(failure_reason, 1)
 
     logger.warning(
         f"Record routed to DLQ: reason={failure_reason}, field={failed_field}",
-        extra={"raw_id": raw_id, "event": "routed_to_dlq"},
+        extra={"raw_id": safe_key, "event": "routed_to_dlq"},
     )
     return True

app/worker/normalizer.py

@@ -113,6 +113,11 @@ def normalize_cdr_payload(
         return None, "malformed_json", None, None
     raw_id = payload.get("raw_id")
     raw_id_str = raw_id if isinstance(raw_id, str) else None
+    if raw_id_str is not None:
+        try:
+            raw_id_str.encode("utf-8")
+        except UnicodeError:
+            raw_id_str = None
     try:
         validate_structure(payload)
     except PayloadError as exc: