feat(worker): complete Phase 3 - processing worker with normalization, micro-batching, and at-least-once offset management
This commit is contained in:
1
tests/__init__.py
Normal file
1
tests/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# tests package
|
||||
287
tests/test_worker.py
Normal file
287
tests/test_worker.py
Normal file
@@ -0,0 +1,287 @@
|
||||
import json
|
||||
import time
|
||||
import unittest
|
||||
import urllib.request
|
||||
|
||||
from app.api.producer import MockMessageProducer
|
||||
from app.worker.clickhouse import MockClickHouseClient
|
||||
from app.worker.consumer import ProcessingWorker
|
||||
from app.worker.internal_server import WorkerInternalServer
|
||||
from app.worker.metrics import worker_metrics
|
||||
from app.worker.normalizer import (
|
||||
NormalizedRecord,
|
||||
normalize_cdr_payload,
|
||||
normalize_phone_number,
|
||||
)
|
||||
|
||||
|
||||
class MockKafkaMessage:
|
||||
def __init__(self, value_bytes: bytes, key: str = "k1", partition: int = 0, offset: int = 0):
|
||||
self._value = value_bytes
|
||||
self._key = key
|
||||
self._partition = partition
|
||||
self._offset = offset
|
||||
|
||||
def value(self):
|
||||
return self._value
|
||||
|
||||
def key(self):
|
||||
return self._key.encode("utf-8") if self._key else None
|
||||
|
||||
def partition(self):
|
||||
return self._partition
|
||||
|
||||
def offset(self):
|
||||
return self._offset
|
||||
|
||||
def error(self):
|
||||
return None
|
||||
|
||||
|
||||
class MockKafkaConsumer:
|
||||
def __init__(self):
|
||||
self.committed = False
|
||||
self.closed = False
|
||||
|
||||
def commit(self, message=None, asynchronous=False):
|
||||
self.committed = True
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
class TestWorkerNormalizer(unittest.TestCase):
|
||||
def test_phone_normalization(self):
|
||||
# 00 prefix
|
||||
self.assertEqual(normalize_phone_number("0019875551234"), "+19875551234")
|
||||
# 011 prefix
|
||||
self.assertEqual(normalize_phone_number("01119875551234"), "+19875551234")
|
||||
# existing + prefix
|
||||
self.assertEqual(normalize_phone_number("+19875551234"), "+19875551234")
|
||||
# invalid
|
||||
self.assertIsNone(normalize_phone_number("invalid-phone"))
|
||||
|
||||
def test_valid_payload_normalization(self):
|
||||
payload = {
|
||||
"raw_id": "raw-test-100",
|
||||
"src_ip": "10.240.12.1",
|
||||
"device_id": "310410998877665",
|
||||
"caller_num": "0019875551234",
|
||||
"svc": "4G_DATA",
|
||||
"up_bytes": 1000,
|
||||
"down_bytes": 2000,
|
||||
"rat": 6,
|
||||
"lac": "4021",
|
||||
"ci": "8812",
|
||||
"epoch_start": 1726070000,
|
||||
"epoch_end": 1726070300,
|
||||
}
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
rec, failure, failed_field, raw_id = normalize_cdr_payload(data)
|
||||
|
||||
self.assertIsNotNone(rec)
|
||||
self.assertIsNone(failure)
|
||||
self.assertEqual(rec.raw_id, "raw-test-100")
|
||||
self.assertEqual(rec.msisdn, "+19875551234")
|
||||
self.assertEqual(rec.radio_access_type, "LTE_4G")
|
||||
self.assertEqual(rec.cell_id, "4021_8812")
|
||||
self.assertEqual(rec.total_bytes, 3000)
|
||||
self.assertEqual(rec.duration_seconds, 300)
|
||||
self.assertEqual(rec.source_ip, "10.240.12.1")
|
||||
self.assertTrue(len(rec.cdr_id) > 0)
|
||||
|
||||
def test_invalid_rat_routing(self):
|
||||
payload = {
|
||||
"raw_id": "raw-test-101",
|
||||
"device_id": "123",
|
||||
"caller_num": "+19875551234",
|
||||
"svc": "4G_DATA",
|
||||
"up_bytes": 10,
|
||||
"down_bytes": 20,
|
||||
"rat": 99, # Unsupported RAT
|
||||
"lac": "1",
|
||||
"ci": "2",
|
||||
"epoch_start": 100,
|
||||
"epoch_end": 200,
|
||||
}
|
||||
rec, failure, failed_field, _ = normalize_cdr_payload(json.dumps(payload).encode("utf-8"))
|
||||
self.assertIsNone(rec)
|
||||
self.assertEqual(failure, "invalid_rat_value")
|
||||
self.assertEqual(failed_field, "rat")
|
||||
|
||||
def test_invalid_epoch_routing(self):
|
||||
payload = {
|
||||
"raw_id": "raw-test-102",
|
||||
"device_id": "123",
|
||||
"caller_num": "+19875551234",
|
||||
"svc": "4G_DATA",
|
||||
"up_bytes": 10,
|
||||
"down_bytes": 20,
|
||||
"rat": 6,
|
||||
"lac": "1",
|
||||
"ci": "2",
|
||||
"epoch_start": 500, # epoch_start > epoch_end
|
||||
"epoch_end": 200,
|
||||
}
|
||||
rec, failure, failed_field, _ = normalize_cdr_payload(json.dumps(payload).encode("utf-8"))
|
||||
self.assertIsNone(rec)
|
||||
self.assertEqual(failure, "invalid_epoch")
|
||||
self.assertEqual(failed_field, "epoch_start")
|
||||
|
||||
def test_negative_bytes_routing(self):
|
||||
payload = {
|
||||
"raw_id": "raw-test-103",
|
||||
"device_id": "123",
|
||||
"caller_num": "+19875551234",
|
||||
"svc": "4G_DATA",
|
||||
"up_bytes": -10, # Negative bytes
|
||||
"down_bytes": 20,
|
||||
"rat": 6,
|
||||
"lac": "1",
|
||||
"ci": "2",
|
||||
"epoch_start": 100,
|
||||
"epoch_end": 200,
|
||||
}
|
||||
rec, failure, failed_field, _ = normalize_cdr_payload(json.dumps(payload).encode("utf-8"))
|
||||
self.assertIsNone(rec)
|
||||
self.assertEqual(failure, "normalization_error")
|
||||
self.assertEqual(failed_field, "up_bytes")
|
||||
|
||||
def test_missing_required_field_routing(self):
|
||||
payload = {
|
||||
"raw_id": "raw-test-104",
|
||||
"device_id": "123",
|
||||
# caller_num missing
|
||||
"svc": "4G_DATA",
|
||||
"up_bytes": 10,
|
||||
"down_bytes": 20,
|
||||
"rat": 6,
|
||||
"lac": "1",
|
||||
"ci": "2",
|
||||
"epoch_start": 100,
|
||||
"epoch_end": 200,
|
||||
}
|
||||
rec, failure, failed_field, _ = normalize_cdr_payload(json.dumps(payload).encode("utf-8"))
|
||||
self.assertIsNone(rec)
|
||||
self.assertEqual(failure, "missing_required_field")
|
||||
self.assertEqual(failed_field, "caller_num")
|
||||
|
||||
|
||||
class TestProcessingWorker(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.consumer = MockKafkaConsumer()
|
||||
self.dlq_producer = MockMessageProducer()
|
||||
self.clickhouse = MockClickHouseClient()
|
||||
self.internal_server = WorkerInternalServer(host="127.0.0.1", port=18001)
|
||||
self.internal_server.start()
|
||||
|
||||
self.worker = ProcessingWorker(
|
||||
consumer=self.consumer,
|
||||
dlq_producer=self.dlq_producer,
|
||||
clickhouse=self.clickhouse,
|
||||
batch_size=2, # Flush on 2 records
|
||||
flush_interval=0.5,
|
||||
internal_server=self.internal_server,
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
self.internal_server.stop()
|
||||
|
||||
def test_micro_batch_flush_on_count(self):
|
||||
payload1 = {
|
||||
"raw_id": "raw-1",
|
||||
"device_id": "1",
|
||||
"caller_num": "+19875551234",
|
||||
"svc": "4G_DATA",
|
||||
"up_bytes": 10,
|
||||
"down_bytes": 20,
|
||||
"rat": 6,
|
||||
"lac": "1",
|
||||
"ci": "2",
|
||||
"epoch_start": 100,
|
||||
"epoch_end": 200,
|
||||
}
|
||||
payload2 = {
|
||||
"raw_id": "raw-2",
|
||||
"device_id": "2",
|
||||
"caller_num": "+19875551234",
|
||||
"svc": "4G_DATA",
|
||||
"up_bytes": 30,
|
||||
"down_bytes": 40,
|
||||
"rat": 6,
|
||||
"lac": "1",
|
||||
"ci": "2",
|
||||
"epoch_start": 100,
|
||||
"epoch_end": 200,
|
||||
}
|
||||
msg1 = MockKafkaMessage(json.dumps(payload1).encode("utf-8"))
|
||||
msg2 = MockKafkaMessage(json.dumps(payload2).encode("utf-8"))
|
||||
|
||||
self.worker.process_single_message(msg1)
|
||||
self.assertEqual(len(self.worker.buffer), 1)
|
||||
self.assertEqual(len(self.clickhouse.rows), 0)
|
||||
|
||||
# Second message reaches batch_size=2 -> flushes to ClickHouse
|
||||
self.worker.process_single_message(msg2)
|
||||
self.assertEqual(len(self.worker.buffer), 0)
|
||||
self.assertEqual(len(self.clickhouse.rows), 2)
|
||||
self.assertTrue(self.consumer.committed)
|
||||
|
||||
def test_dlq_routing_for_malformed_message(self):
|
||||
bad_msg = MockKafkaMessage(b"{not json")
|
||||
self.worker.process_single_message(bad_msg)
|
||||
|
||||
self.assertEqual(len(self.dlq_producer.messages), 1)
|
||||
dlq_data = json.loads(self.dlq_producer.messages[0]["value"].decode("utf-8"))
|
||||
self.assertEqual(dlq_data["failure_reason"], "malformed_json")
|
||||
|
||||
def test_clickhouse_retry_and_dlq_fallback(self):
|
||||
# ClickHouse configured to fail 3 times (exceeding max_attempts=3)
|
||||
ch_failing = MockClickHouseClient(fail_attempts=5)
|
||||
worker = ProcessingWorker(
|
||||
consumer=self.consumer,
|
||||
dlq_producer=self.dlq_producer,
|
||||
clickhouse=ch_failing,
|
||||
batch_size=1,
|
||||
flush_interval=1.0,
|
||||
internal_server=self.internal_server,
|
||||
)
|
||||
payload = {
|
||||
"raw_id": "raw-ch-fail",
|
||||
"device_id": "1",
|
||||
"caller_num": "+19875551234",
|
||||
"svc": "4G_DATA",
|
||||
"up_bytes": 10,
|
||||
"down_bytes": 20,
|
||||
"rat": 6,
|
||||
"lac": "1",
|
||||
"ci": "2",
|
||||
"epoch_start": 100,
|
||||
"epoch_end": 200,
|
||||
}
|
||||
msg = MockKafkaMessage(json.dumps(payload).encode("utf-8"))
|
||||
worker.process_single_message(msg)
|
||||
|
||||
# Record should have fallen back to DLQ with ch_insert_failed
|
||||
self.assertEqual(len(self.dlq_producer.messages), 1)
|
||||
dlq_data = json.loads(self.dlq_producer.messages[0]["value"].decode("utf-8"))
|
||||
self.assertEqual(dlq_data["failure_reason"], "ch_insert_failed")
|
||||
|
||||
def test_internal_stats_and_metrics_endpoints(self):
|
||||
# Verify /internal/stats
|
||||
with urllib.request.urlopen("http://127.0.0.1:18001/internal/stats") as resp:
|
||||
self.assertEqual(resp.status, 200)
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
self.assertIn("pipeline", data)
|
||||
self.assertIn("breakdown", data)
|
||||
|
||||
# Verify /metrics
|
||||
with urllib.request.urlopen("http://127.0.0.1:18001/metrics") as resp:
|
||||
self.assertEqual(resp.status, 200)
|
||||
text = resp.read().decode("utf-8")
|
||||
self.assertIn("cdr_processed_total", text)
|
||||
self.assertIn("cdr_dlq_total", text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user