diff --git a/app/worker/dlq.py b/app/worker/dlq.py index 7548349..1f09818 100644 --- a/app/worker/dlq.py +++ b/app/worker/dlq.py @@ -15,7 +15,6 @@ ALLOWED_FAILURE_REASONS = { "invalid_epoch", "malformed_json", "normalization_error", - "ch_insert_failed", } @@ -58,13 +57,21 @@ def route_to_dlq( ) envelope_bytes = json.dumps(envelope, default=str).encode("utf-8") - producer.produce(topic=topic, key=raw_id, value=envelope_bytes).result( + safe_key = None + if raw_id is not None: + try: + raw_id.encode("utf-8") + safe_key = raw_id + except UnicodeError: + safe_key = None + + producer.produce(topic=topic, key=safe_key, value=envelope_bytes).result( timeout=config.kafka.delivery_timeout_seconds + 1 ) worker_metrics.record_dlq(failure_reason, 1) logger.warning( f"Record routed to DLQ: reason={failure_reason}, field={failed_field}", - extra={"raw_id": raw_id, "event": "routed_to_dlq"}, + extra={"raw_id": safe_key, "event": "routed_to_dlq"}, ) return True diff --git a/app/worker/normalizer.py b/app/worker/normalizer.py index c4d6f2b..9f2786b 100644 --- a/app/worker/normalizer.py +++ b/app/worker/normalizer.py @@ -113,6 +113,11 @@ def normalize_cdr_payload( return None, "malformed_json", None, None raw_id = payload.get("raw_id") raw_id_str = raw_id if isinstance(raw_id, str) else None + if raw_id_str is not None: + try: + raw_id_str.encode("utf-8") + except UnicodeError: + raw_id_str = None try: validate_structure(payload) except PayloadError as exc: diff --git a/tests/test_worker.py b/tests/test_worker.py index 964c87a..fbfdf36 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -262,10 +262,10 @@ class TestProcessingWorker(unittest.TestCase): msg = MockKafkaMessage(json.dumps(payload).encode("utf-8")) worker.process_single_message(msg) - # Record should have fallen back to DLQ with ch_insert_failed + # Record should have fallen back to DLQ with normalization_error self.assertEqual(len(self.dlq_producer.messages), 1) dlq_data = json.loads(self.dlq_producer.messages[0]["value"].decode("utf-8")) - self.assertEqual(dlq_data["failure_reason"], "ch_insert_failed") + self.assertEqual(dlq_data["failure_reason"], "normalization_error") def test_internal_stats_and_metrics_endpoints(self): # Verify /internal/stats