288 lines
9.6 KiB
Python
288 lines
9.6 KiB
Python
import json
|
|
import time
|
|
import unittest
|
|
import urllib.request
|
|
|
|
from app.api.producer import MockMessageProducer
|
|
from app.worker.clickhouse import MockClickHouseClient
|
|
from app.worker.consumer import ProcessingWorker
|
|
from app.worker.internal_server import WorkerInternalServer
|
|
from app.worker.metrics import worker_metrics
|
|
from app.worker.normalizer import (
|
|
NormalizedRecord,
|
|
normalize_cdr_payload,
|
|
normalize_phone_number,
|
|
)
|
|
|
|
|
|
class MockKafkaMessage:
|
|
def __init__(self, value_bytes: bytes, key: str = "k1", partition: int = 0, offset: int = 0):
|
|
self._value = value_bytes
|
|
self._key = key
|
|
self._partition = partition
|
|
self._offset = offset
|
|
|
|
def value(self):
|
|
return self._value
|
|
|
|
def key(self):
|
|
return self._key.encode("utf-8") if self._key else None
|
|
|
|
def partition(self):
|
|
return self._partition
|
|
|
|
def offset(self):
|
|
return self._offset
|
|
|
|
def error(self):
|
|
return None
|
|
|
|
|
|
class MockKafkaConsumer:
|
|
def __init__(self):
|
|
self.committed = False
|
|
self.closed = False
|
|
|
|
def commit(self, message=None, offsets=None, asynchronous=False):
|
|
self.committed = True
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
class TestWorkerNormalizer(unittest.TestCase):
|
|
def test_phone_normalization(self):
|
|
# 00 prefix
|
|
self.assertEqual(normalize_phone_number("0019875551234"), "+19875551234")
|
|
# 011 prefix
|
|
self.assertEqual(normalize_phone_number("01119875551234"), "+19875551234")
|
|
# existing + prefix
|
|
self.assertEqual(normalize_phone_number("+19875551234"), "+19875551234")
|
|
# invalid
|
|
self.assertIsNone(normalize_phone_number("invalid-phone"))
|
|
|
|
def test_valid_payload_normalization(self):
|
|
payload = {
|
|
"raw_id": "raw-test-100",
|
|
"src_ip": "10.240.12.1",
|
|
"device_id": "310410998877665",
|
|
"caller_num": "0019875551234",
|
|
"svc": "4G_DATA",
|
|
"up_bytes": 1000,
|
|
"down_bytes": 2000,
|
|
"rat": 6,
|
|
"lac": "4021",
|
|
"ci": "8812",
|
|
"epoch_start": 1726070000,
|
|
"epoch_end": 1726070300,
|
|
}
|
|
data = json.dumps(payload).encode("utf-8")
|
|
rec, failure, failed_field, raw_id = normalize_cdr_payload(data)
|
|
|
|
self.assertIsNotNone(rec)
|
|
self.assertIsNone(failure)
|
|
self.assertEqual(rec.raw_id, "raw-test-100")
|
|
self.assertEqual(rec.msisdn, "+19875551234")
|
|
self.assertEqual(rec.radio_access_type, "LTE_4G")
|
|
self.assertEqual(rec.cell_id, "4021_8812")
|
|
self.assertEqual(rec.total_bytes, 3000)
|
|
self.assertEqual(rec.duration_seconds, 300)
|
|
self.assertEqual(rec.source_ip, "10.240.12.1")
|
|
self.assertTrue(len(rec.cdr_id) > 0)
|
|
|
|
def test_invalid_rat_routing(self):
|
|
payload = {
|
|
"raw_id": "raw-test-101",
|
|
"device_id": "123",
|
|
"caller_num": "+19875551234",
|
|
"svc": "4G_DATA",
|
|
"up_bytes": 10,
|
|
"down_bytes": 20,
|
|
"rat": 99, # Unsupported RAT
|
|
"lac": "1",
|
|
"ci": "2",
|
|
"epoch_start": 100,
|
|
"epoch_end": 200,
|
|
}
|
|
rec, failure, failed_field, _ = normalize_cdr_payload(json.dumps(payload).encode("utf-8"))
|
|
self.assertIsNone(rec)
|
|
self.assertEqual(failure, "invalid_rat_value")
|
|
self.assertEqual(failed_field, "rat")
|
|
|
|
def test_invalid_epoch_routing(self):
|
|
payload = {
|
|
"raw_id": "raw-test-102",
|
|
"device_id": "123",
|
|
"caller_num": "+19875551234",
|
|
"svc": "4G_DATA",
|
|
"up_bytes": 10,
|
|
"down_bytes": 20,
|
|
"rat": 6,
|
|
"lac": "1",
|
|
"ci": "2",
|
|
"epoch_start": 500, # epoch_start > epoch_end
|
|
"epoch_end": 200,
|
|
}
|
|
rec, failure, failed_field, _ = normalize_cdr_payload(json.dumps(payload).encode("utf-8"))
|
|
self.assertIsNone(rec)
|
|
self.assertEqual(failure, "invalid_epoch")
|
|
self.assertEqual(failed_field, "epoch_start")
|
|
|
|
def test_negative_bytes_routing(self):
|
|
payload = {
|
|
"raw_id": "raw-test-103",
|
|
"device_id": "123",
|
|
"caller_num": "+19875551234",
|
|
"svc": "4G_DATA",
|
|
"up_bytes": -10, # Negative bytes
|
|
"down_bytes": 20,
|
|
"rat": 6,
|
|
"lac": "1",
|
|
"ci": "2",
|
|
"epoch_start": 100,
|
|
"epoch_end": 200,
|
|
}
|
|
rec, failure, failed_field, _ = normalize_cdr_payload(json.dumps(payload).encode("utf-8"))
|
|
self.assertIsNone(rec)
|
|
self.assertEqual(failure, "normalization_error")
|
|
self.assertEqual(failed_field, "up_bytes")
|
|
|
|
def test_missing_required_field_routing(self):
|
|
payload = {
|
|
"raw_id": "raw-test-104",
|
|
"device_id": "123",
|
|
# caller_num missing
|
|
"svc": "4G_DATA",
|
|
"up_bytes": 10,
|
|
"down_bytes": 20,
|
|
"rat": 6,
|
|
"lac": "1",
|
|
"ci": "2",
|
|
"epoch_start": 100,
|
|
"epoch_end": 200,
|
|
}
|
|
rec, failure, failed_field, _ = normalize_cdr_payload(json.dumps(payload).encode("utf-8"))
|
|
self.assertIsNone(rec)
|
|
self.assertEqual(failure, "missing_required_field")
|
|
self.assertEqual(failed_field, "caller_num")
|
|
|
|
|
|
class TestProcessingWorker(unittest.TestCase):
|
|
def setUp(self):
|
|
self.consumer = MockKafkaConsumer()
|
|
self.dlq_producer = MockMessageProducer()
|
|
self.clickhouse = MockClickHouseClient()
|
|
self.internal_server = WorkerInternalServer(host="127.0.0.1", port=18001)
|
|
self.internal_server.start()
|
|
|
|
self.worker = ProcessingWorker(
|
|
consumer=self.consumer,
|
|
dlq_producer=self.dlq_producer,
|
|
clickhouse=self.clickhouse,
|
|
batch_size=2, # Flush on 2 records
|
|
flush_interval=0.5,
|
|
internal_server=self.internal_server,
|
|
)
|
|
|
|
def tearDown(self):
|
|
self.internal_server.stop()
|
|
|
|
def test_micro_batch_flush_on_count(self):
|
|
payload1 = {
|
|
"raw_id": "raw-1",
|
|
"device_id": "1",
|
|
"caller_num": "+19875551234",
|
|
"svc": "4G_DATA",
|
|
"up_bytes": 10,
|
|
"down_bytes": 20,
|
|
"rat": 6,
|
|
"lac": "1",
|
|
"ci": "2",
|
|
"epoch_start": 100,
|
|
"epoch_end": 200,
|
|
}
|
|
payload2 = {
|
|
"raw_id": "raw-2",
|
|
"device_id": "2",
|
|
"caller_num": "+19875551234",
|
|
"svc": "4G_DATA",
|
|
"up_bytes": 30,
|
|
"down_bytes": 40,
|
|
"rat": 6,
|
|
"lac": "1",
|
|
"ci": "2",
|
|
"epoch_start": 100,
|
|
"epoch_end": 200,
|
|
}
|
|
msg1 = MockKafkaMessage(json.dumps(payload1).encode("utf-8"))
|
|
msg2 = MockKafkaMessage(json.dumps(payload2).encode("utf-8"))
|
|
|
|
self.worker.process_single_message(msg1)
|
|
self.assertEqual(len(self.worker.buffer), 1)
|
|
self.assertEqual(len(self.clickhouse.rows), 0)
|
|
|
|
# Second message reaches batch_size=2 -> flushes to ClickHouse
|
|
self.worker.process_single_message(msg2)
|
|
self.assertEqual(len(self.worker.buffer), 0)
|
|
self.assertEqual(len(self.clickhouse.rows), 2)
|
|
self.assertTrue(self.consumer.committed)
|
|
|
|
def test_dlq_routing_for_malformed_message(self):
|
|
bad_msg = MockKafkaMessage(b"{not json")
|
|
self.worker.process_single_message(bad_msg)
|
|
|
|
self.assertEqual(len(self.dlq_producer.messages), 1)
|
|
dlq_data = json.loads(self.dlq_producer.messages[0]["value"].decode("utf-8"))
|
|
self.assertEqual(dlq_data["failure_reason"], "malformed_json")
|
|
|
|
def test_clickhouse_retry_and_dlq_fallback(self):
|
|
# ClickHouse configured to fail 3 times (exceeding max_attempts=3)
|
|
ch_failing = MockClickHouseClient(fail_attempts=5)
|
|
worker = ProcessingWorker(
|
|
consumer=self.consumer,
|
|
dlq_producer=self.dlq_producer,
|
|
clickhouse=ch_failing,
|
|
batch_size=1,
|
|
flush_interval=1.0,
|
|
internal_server=self.internal_server,
|
|
)
|
|
payload = {
|
|
"raw_id": "raw-ch-fail",
|
|
"device_id": "1",
|
|
"caller_num": "+19875551234",
|
|
"svc": "4G_DATA",
|
|
"up_bytes": 10,
|
|
"down_bytes": 20,
|
|
"rat": 6,
|
|
"lac": "1",
|
|
"ci": "2",
|
|
"epoch_start": 100,
|
|
"epoch_end": 200,
|
|
}
|
|
msg = MockKafkaMessage(json.dumps(payload).encode("utf-8"))
|
|
worker.process_single_message(msg)
|
|
|
|
# Record should have fallen back to DLQ with normalization_error
|
|
self.assertEqual(len(self.dlq_producer.messages), 1)
|
|
dlq_data = json.loads(self.dlq_producer.messages[0]["value"].decode("utf-8"))
|
|
self.assertEqual(dlq_data["failure_reason"], "normalization_error")
|
|
|
|
def test_internal_stats_and_metrics_endpoints(self):
|
|
# Verify /internal/stats
|
|
with urllib.request.urlopen("http://127.0.0.1:18001/internal/stats") as resp:
|
|
self.assertEqual(resp.status, 200)
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
self.assertIn("pipeline", data)
|
|
self.assertIn("breakdown", data)
|
|
|
|
# Verify /metrics
|
|
with urllib.request.urlopen("http://127.0.0.1:18001/metrics") as resp:
|
|
self.assertEqual(resp.status, 200)
|
|
text = resp.read().decode("utf-8")
|
|
self.assertIn("cdr_processed_total", text)
|
|
self.assertIn("cdr_dlq_total", text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|