import os import base64 import argparse import json import random import re import sys import time import uuid import urllib.error import urllib.parse import urllib.request from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass from datetime import datetime, timezone from typing import Any, Dict, Iterable, List, Optional, Tuple RAT_CHOICES = [1, 6, 9] # 3G, LTE_4G, 5G_NR CLICKHOUSE_DATABASE = os.getenv("CLICKHOUSE_DATABASE", "telecom") CLICKHOUSE_TABLE = os.getenv("CLICKHOUSE_TABLE", "cdr_records") SERVICE_CHOICES = ["4G_DATA", "5G_DATA", "VOICE_CALL", "SMS"] PHONE_PREFIXES = ["001", "0111", "+1"] sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @dataclass class HttpResult: status: int body: bytes elapsed_ms: float @dataclass class ScenarioResult: name: str passed: bool detail: str def send_http_request( url: str, method: str = "GET", body: bytes = b"", headers: Optional[Dict[str, str]] = None, timeout: float = 15.0, ) -> HttpResult: req = urllib.request.Request( url, data=body if method in ("POST", "PUT", "PATCH") else None, headers=headers or {}, method=method, ) started = time.perf_counter() try: with urllib.request.urlopen(req, timeout=timeout) as resp: return HttpResult( status=resp.status, body=resp.read(), elapsed_ms=(time.perf_counter() - started) * 1000, ) except urllib.error.HTTPError as exc: return HttpResult( status=exc.code, body=exc.read(), elapsed_ms=(time.perf_counter() - started) * 1000, ) except Exception as exc: return HttpResult( status=0, body=str(exc).encode("utf-8", errors="replace"), elapsed_ms=(time.perf_counter() - started) * 1000, ) def generate_valid_record(raw_id: str, sequence: int = 0) -> Dict[str, Any]: """Generate a valid record while deterministically covering all RAT/service variants.""" now = int(time.time()) start = now - random.randint(100, 1000) end = start + random.randint(10, 600) rat = RAT_CHOICES[sequence % len(RAT_CHOICES)] svc = SERVICE_CHOICES[sequence % len(SERVICE_CHOICES)] phone_prefix = PHONE_PREFIXES[sequence % len(PHONE_PREFIXES)] return { "raw_id": raw_id, "src_ip": f"10.240.{random.randint(1, 254)}.{random.randint(1, 254)}", "device_id": f"310410{random.randint(100000000, 999999999)}", "caller_num": f"{phone_prefix}987555{random.randint(1000, 9999)}", "svc": svc, "up_bytes": random.randint(1000, 5_000_000), "down_bytes": random.randint(1000, 20_000_000), "rat": rat, "lac": str(random.randint(1000, 9999)), "ci": str(random.randint(1000, 9999)), "epoch_start": start, "epoch_end": end, } def post_json(endpoint: str, payload: Dict[str, Any], headers: Dict[str, str]) -> HttpResult: return send_http_request( endpoint, method="POST", body=json.dumps(payload).encode("utf-8"), headers=headers, ) def status_counts(results: Iterable[HttpResult]) -> Counter: return Counter(result.status for result in results) def format_status_counts(counts: Counter) -> str: if not counts: return "none" return ", ".join(f"{status}:{count}" for status, count in sorted(counts.items())) def is_structured_json_error(result: HttpResult) -> bool: if result.status < 400: return False try: payload = json.loads(result.body.decode("utf-8")) except Exception: return False return isinstance(payload, dict) and bool(payload.get("error")) def fetch_json(url: str) -> Optional[Dict[str, Any]]: result = send_http_request(url) if result.status != 200: return None try: payload = json.loads(result.body.decode("utf-8")) return payload if isinstance(payload, dict) else None except Exception: return None def stats_snapshot(stats_url: str) -> Optional[Dict[str, Any]]: data = fetch_json(stats_url) if data is None: return None pipeline = data.get("pipeline", {}) if isinstance(data.get("pipeline"), dict) else {} if pipeline.get("worker_status") == "unreachable": return None breakdown = data.get("breakdown", {}) if isinstance(data.get("breakdown"), dict) else {} dlq = breakdown.get("dlq_by_reason", {}) if isinstance(breakdown.get("dlq_by_reason"), dict) else {} return { "total_received": int(pipeline.get("total_received", 0) or 0), "total_processed": int(pipeline.get("total_processed", 0) or 0), "total_dlq": int(pipeline.get("total_dlq", 0) or 0), "in_flight_buffer": int(pipeline.get("in_flight_buffer", 0) or 0), "dlq_by_reason": {str(k): int(v or 0) for k, v in dlq.items()}, } def delta(after: int, before: int) -> int: return after - before def dlq_reason_delta( before: Optional[Dict[str, Any]], after: Optional[Dict[str, Any]], reason: str ) -> Optional[int]: if before is None or after is None: return None before_value = before.get("dlq_by_reason", {}).get(reason, 0) after_value = after.get("dlq_by_reason", {}).get(reason, 0) return delta(int(after_value), int(before_value)) def clickhouse_query(clickhouse_url: str, query: str) -> Tuple[bool, str]: parsed = urllib.parse.urlsplit(clickhouse_url) query_params = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True) query_params.append(("query", query)) url = urllib.parse.urlunsplit(parsed._replace(path=parsed.path or "/", query=urllib.parse.urlencode(query_params))) auth = base64.b64encode(f"{os.getenv('CLICKHOUSE_USER', 'default')}:{os.getenv('CLICKHOUSE_PASSWORD', '')}".encode()).decode() result = send_http_request(url, headers={"Authorization": f"Basic {auth}"}, timeout=10.0) if result.status != 200: return False, result.body.decode("utf-8", errors="replace").strip() return True, result.body.decode("utf-8", errors="replace").strip() def ch_count(clickhouse_url: str, where_clause: str) -> Optional[int]: ok, body = clickhouse_query( clickhouse_url, f"SELECT count() FROM {CLICKHOUSE_DATABASE}.{CLICKHOUSE_TABLE} FINAL WHERE {where_clause}", ) if not ok: return None try: return int(body.splitlines()[0].strip()) except Exception: return None def quote_ch_string(value: str) -> str: return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" def verify_normalized_rows(clickhouse_url, run_prefix, payloads): expected = {payload["raw_id"]: payload for payload in payloads} query = (f"SELECT *, toUnixTimestamp(session_start_time) AS start_epoch, " f"toUnixTimestamp(session_end_time) AS end_epoch " f"FROM {CLICKHOUSE_DATABASE}.{CLICKHOUSE_TABLE} FINAL " f"WHERE startsWith(raw_id, {quote_ch_string(run_prefix)}) FORMAT JSONEachRow") ok, body = clickhouse_query(clickhouse_url, query) if not ok: return False, "ClickHouse rows unavailable" rows = [json.loads(line) for line in body.splitlines() if line.strip()] if len(rows) != len(expected) or {row["raw_id"] for row in rows} != set(expected): return False, "Stored identities differ from submitted identities" rat_map = {1: "3G", 6: "LTE_4G", 9: "5G_NR"} for row in rows: raw = expected[row["raw_id"]] phone = raw["caller_num"] phone = "+" + phone[2:] if phone.startswith("00") else "+" + phone[3:] if phone.startswith("011") else phone strings = {"cdr_id": str(uuid.uuid5(uuid.NAMESPACE_DNS, raw["raw_id"])), "msisdn": phone, "imsi": raw["device_id"], "service_type": raw["svc"], "radio_access_type": rat_map[raw["rat"]], "cell_id": f"{raw['lac']}_{raw['ci']}", "source_ip": raw.get("src_ip") or "0.0.0.0"} numbers = {"bytes_uploaded": raw["up_bytes"], "bytes_downloaded": raw["down_bytes"], "total_bytes": raw["up_bytes"] + raw["down_bytes"], "duration_seconds": raw["epoch_end"] - raw["epoch_start"], "start_epoch": raw["epoch_start"], "end_epoch": raw["epoch_end"]} if any(row.get(key) != value for key, value in strings.items()) or any(int(row[key]) != value for key, value in numbers.items()): return False, f"Incorrect transformation for {raw['raw_id']}" return True, f"All {len(rows)} identities and normalized field values match" def wait_for_pipeline( clickhouse_url: str, stats_url: str, run_prefix: str, expected_clickhouse: int, expected_dlq_delta: int, baseline_stats: Optional[Dict[str, Any]], timeout_seconds: float, ) -> Tuple[Optional[int], Optional[Dict[str, Any]], float]: started = time.monotonic() last_count: Optional[int] = None last_stats: Optional[Dict[str, Any]] = None prefix_literal = quote_ch_string(run_prefix) while True: last_count = ch_count(clickhouse_url, f"startsWith(raw_id, {prefix_literal})") last_stats = stats_snapshot(stats_url) ch_ready = last_count is not None and last_count >= expected_clickhouse if baseline_stats is not None and last_stats is not None: dlq_ready = delta(last_stats["total_dlq"], baseline_stats["total_dlq"]) >= expected_dlq_delta buffer_ready = last_stats.get("in_flight_buffer", 0) == 0 else: # We still wait for ClickHouse if /stats is unavailable, then report observability separately. dlq_ready = True buffer_ready = True if ch_ready and dlq_ready and buffer_ready: break if time.monotonic() - started >= timeout_seconds: break time.sleep(0.5) return last_count, last_stats, time.monotonic() - started def run_verification( api_url: str, clickhouse_url: str, stats_url: str, settle_timeout: float, burst_workers: int, kafka_brokers: str = "localhost:9092", raw_topic: str = "cdr.raw", dlq_topic: str = "cdr.raw.dlq", ) -> int: run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + f"-{random.randint(1000, 9999)}" run_prefix = f"verify-{run_id}-" ingest_endpoint = f"{api_url.rstrip('/')}/api/v1/cdr/raw" headers = {"Content-Type": "application/json"} try: from verification_kafka import KafkaEvidence for name in (CLICKHOUSE_DATABASE, CLICKHOUSE_TABLE): if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): raise ValueError(f"Invalid ClickHouse identifier: {name!r}") evidence = KafkaEvidence(kafka_brokers, raw_topic, dlq_topic, run_prefix) except Exception as exc: print(f"OVERALL: FAIL - actual Kafka evidence is required: {exc}") return 1 try: print("\n============================================================") print(" Telecom CDR Pipeline - Assessment Verification") print(f" Run ID: {run_id}") print(f" Ingestion API: {ingest_endpoint}") print(f" Stats endpoint: {stats_url}") print(f" ClickHouse endpoint: {clickhouse_url}") print("============================================================\n") baseline_stats = stats_snapshot(stats_url) if baseline_stats is None: print("WARN: /stats was not readable before the test; DLQ/stat deltas cannot be verified reliably.\n") total_sent = 0 all_statuses: Counter = Counter() scenario_http: Dict[str, List[HttpResult]] = {} valid_payloads: List[Dict[str, Any]] = [] # 1. 1,000 valid records. print("[1/7] 1,000 valid records (varied RAT and service type)") valid_results: List[HttpResult] = [] for i in range(1000): payload = generate_valid_record(f"{run_prefix}valid-{i + 1:05d}", i) if i == 1: # Same metadata, different identity: both must survive replacement. payload = valid_payloads[0] | {"raw_id": payload["raw_id"]} valid_payloads.append(payload) result = post_json(ingest_endpoint, payload, headers) valid_results.append(result) scenario_http["valid"] = valid_results counts = status_counts(valid_results) all_statuses.update(counts) total_sent += len(valid_results) print(f" HTTP statuses: {format_status_counts(counts)}") # 2. Missing required caller_num. print("[2/7] 20 records missing required field 'caller_num'") missing_results: List[HttpResult] = [] for i in range(20): payload = generate_valid_record(f"{run_prefix}missing-{i + 1:04d}", i) del payload["caller_num"] missing_results.append(post_json(ingest_endpoint, payload, headers)) scenario_http["missing"] = missing_results counts = status_counts(missing_results) all_statuses.update(counts) total_sent += len(missing_results) print(f" HTTP statuses: {format_status_counts(counts)}") for i in range(20): direct = generate_valid_record(f"{run_prefix}direct-missing-{i:04d}", i) del direct["caller_num"] evidence.inject(direct["raw_id"], json.dumps(direct).encode(), "missing_required_field") # 3. Invalid JSON must be rejected by the API with structured 400 errors. print("[3/7] 10 malformed JSON requests") malformed_results: List[HttpResult] = [] for i in range(10): body = b'{"raw_id":"' + f"{run_prefix}bad-json-{i + 1}".encode("utf-8") + b'","broken":' malformed_results.append( send_http_request(ingest_endpoint, "POST", body, headers) ) scenario_http["malformed"] = malformed_results counts = status_counts(malformed_results) all_statuses.update(counts) total_sent += len(malformed_results) structured_errors = sum(is_structured_json_error(r) for r in malformed_results) print(f" HTTP statuses: {format_status_counts(counts)}; structured JSON errors: {structured_errors}/10") for i in range(10): key = f"{run_prefix}direct-json-{i:04d}" evidence.inject(key, ('{"raw_id":"' + key + '","broken":').encode(), "malformed_json") # 4. Unknown RAT should be accepted into the pipeline then routed to DLQ by worker. print("[4/7] 10 records with unknown RAT (rat=99)") bad_rat_results: List[HttpResult] = [] for i in range(10): payload = generate_valid_record(f"{run_prefix}bad-rat-{i + 1:04d}", i) payload["rat"] = 99 evidence.expected[payload["raw_id"]] = (json.dumps(payload).encode(), "invalid_rat_value") bad_rat_results.append(post_json(ingest_endpoint, payload, headers)) scenario_http["bad_rat"] = bad_rat_results counts = status_counts(bad_rat_results) all_statuses.update(counts) total_sent += len(bad_rat_results) print(f" HTTP statuses: {format_status_counts(counts)}") # 5. Inverted epoch should be routed to DLQ by worker. print("[5/7] 10 records with epoch_start > epoch_end") bad_epoch_results: List[HttpResult] = [] for i in range(10): payload = generate_valid_record(f"{run_prefix}bad-epoch-{i + 1:04d}", i) payload["epoch_start"] = payload["epoch_end"] + 500 evidence.expected[payload["raw_id"]] = (json.dumps(payload).encode(), "invalid_epoch") bad_epoch_results.append(post_json(ingest_endpoint, payload, headers)) scenario_http["bad_epoch"] = bad_epoch_results counts = status_counts(bad_epoch_results) all_statuses.update(counts) total_sent += len(bad_epoch_results) print(f" HTTP statuses: {format_status_counts(counts)}") # 6. Replay the exact same first 20 valid payloads, not newly-generated data. print("[6/7] Replay 20 exact duplicate records (same raw_id and same payload)") duplicate_results = [post_json(ingest_endpoint, payload, headers) for payload in valid_payloads[:20]] scenario_http["duplicates"] = duplicate_results counts = status_counts(duplicate_results) all_statuses.update(counts) total_sent += len(duplicate_results) print(f" HTTP statuses: {format_status_counts(counts)}") # 7. 500-record burst; wall-clock requirement is <5 seconds and no 429. print(f"[7/7] 500-record concurrent burst (<5s, no 429), workers={burst_workers}") burst_payloads = [ generate_valid_record(f"{run_prefix}burst-{i + 1:04d}", i) for i in range(500) ] burst_started = time.perf_counter() burst_results: List[HttpResult] = [] with ThreadPoolExecutor(max_workers=burst_workers) as executor: futures = [executor.submit(post_json, ingest_endpoint, p, headers) for p in burst_payloads] for future in as_completed(futures): burst_results.append(future.result()) burst_duration = time.perf_counter() - burst_started scenario_http["burst"] = burst_results counts = status_counts(burst_results) all_statuses.update(counts) total_sent += len(burst_results) print(f" Duration: {burst_duration:.3f}s; HTTP statuses: {format_status_counts(counts)}") print(f"\nWaiting for worker flush / DLQ processing (up to {settle_timeout:.1f}s)...") expected_dlq_delta = 50 ch_total, final_stats, settle_elapsed = wait_for_pipeline( clickhouse_url=clickhouse_url, stats_url=stats_url, run_prefix=run_prefix, expected_clickhouse=1500, expected_dlq_delta=expected_dlq_delta, baseline_stats=baseline_stats, timeout_seconds=settle_timeout, ) print(f"Pipeline verification wait: {settle_elapsed:.2f}s\n") prefix_literal = quote_ch_string(run_prefix) valid_ch = ch_count(clickhouse_url, f"startsWith(raw_id, {quote_ch_string(run_prefix + 'valid-')})") burst_ch = ch_count(clickhouse_url, f"startsWith(raw_id, {quote_ch_string(run_prefix + 'burst-')})") duplicate_ids = [payload["raw_id"] for payload in valid_payloads[:20]] duplicate_list = ",".join(quote_ch_string(raw_id) for raw_id in duplicate_ids) duplicate_unique_ch = ch_count(clickhouse_url, f"raw_id IN ({duplicate_list})") dlq_records = evidence.collect(50, settle_timeout) dlq_verified = evidence.validate() reasons = Counter(record["failure_reason"] for record in dlq_records) missing_dlq = reasons["missing_required_field"] bad_rat_dlq = reasons["invalid_rat_value"] bad_epoch_dlq = reasons["invalid_epoch"] results: List[ScenarioResult] = [] valid_http_ok = status_counts(valid_results) == Counter({202: 1000}) results.append( ScenarioResult( "1. Valid records", valid_http_ok and valid_ch == 1000, f"HTTP 202={sum(r.status == 202 for r in valid_results)}/1000; ClickHouse={valid_ch}/1000", ) ) # HTTP rejection and direct Kafka DLQ delivery are independent requirements. missing_transport_ok = all(r.status == 400 and is_structured_json_error(r) for r in missing_results) missing_passed = missing_transport_ok and missing_dlq == 20 results.append( ScenarioResult( "2. Missing caller_num", missing_passed, f"HTTP={format_status_counts(status_counts(missing_results))}; DLQ missing_required_field={missing_dlq}/20", ) ) malformed_400 = sum(r.status == 400 for r in malformed_results) results.append( ScenarioResult( "3. Malformed JSON", malformed_400 == 10 and structured_errors == 10, f"HTTP 400={malformed_400}/10; structured JSON errors={structured_errors}/10", ) ) results.append( ScenarioResult( "4. Invalid RAT", all(r.status == 202 for r in bad_rat_results) and bad_rat_dlq == 10, f"HTTP 202={sum(r.status == 202 for r in bad_rat_results)}/10; DLQ invalid_rat_value={bad_rat_dlq if bad_rat_dlq is not None else 'unavailable'}/10", ) ) results.append( ScenarioResult( "5. Invalid epoch", all(r.status == 202 for r in bad_epoch_results) and bad_epoch_dlq == 10, f"HTTP 202={sum(r.status == 202 for r in bad_epoch_results)}/10; DLQ invalid_epoch={bad_epoch_dlq if bad_epoch_dlq is not None else 'unavailable'}/10", ) ) results.append( ScenarioResult( "6. Duplicate raw_id", all(r.status == 202 for r in duplicate_results) and duplicate_unique_ch == 20, f"duplicate POSTs accepted={sum(r.status == 202 for r in duplicate_results)}/20; unique rows after FINAL={duplicate_unique_ch}/20", ) ) burst_429 = sum(r.status == 429 for r in burst_results) burst_202 = sum(r.status == 202 for r in burst_results) burst_network_errors = sum(r.status == 0 for r in burst_results) results.append( ScenarioResult( "7. 500-record burst", burst_duration < 5.0 and burst_202 == 500 and burst_429 == 0 and burst_network_errors == 0 and burst_ch == 500, f"duration={burst_duration:.3f}s; 202={burst_202}/500; 429={burst_429}; network_errors={burst_network_errors}; ClickHouse={burst_ch}/500", ) ) results.append(ScenarioResult("Actual DLQ envelopes", dlq_verified and len(dlq_records) == 50, f"verified={dlq_verified}; records={len(dlq_records)}/50; reasons={dict(reasons)}")) metrics_ok = False if baseline_stats is not None and final_stats is not None: expected_dlq_metric = 50 metrics_ok = (delta(final_stats["total_received"], baseline_stats["total_received"]) == 1540 and delta(final_stats["total_processed"], baseline_stats["total_processed"]) == 1520 and delta(final_stats["total_dlq"], baseline_stats["total_dlq"]) == expected_dlq_metric) results.append(ScenarioResult("Acknowledged metrics", metrics_ok, f"received=1540; processed=1520; DLQ=50 expected")) storage_ok, storage_detail = verify_normalized_rows(clickhouse_url, run_prefix, valid_payloads + burst_payloads) results.append(ScenarioResult("Normalized storage", storage_ok, storage_detail)) print("=== Assessment Verification Summary ===") print(f"Run prefix: {run_prefix}") print(f"Records/requests sent: {total_sent} (expected 1,570)") print(f"All HTTP statuses: {format_status_counts(all_statuses)}") print(f"Run-scoped ClickHouse rows: {ch_total if ch_total is not None else 'unavailable'} (expected 1,500 unique valid+burst rows)") if baseline_stats is not None and final_stats is not None: print(f"/stats received delta: {delta(final_stats['total_received'], baseline_stats['total_received'])}") print(f"/stats processed delta: {delta(final_stats['total_processed'], baseline_stats['total_processed'])}") print(f"/stats DLQ delta: {delta(final_stats['total_dlq'], baseline_stats['total_dlq'])} (expected 50)") print(f"/stats in-flight buffer: {final_stats.get('in_flight_buffer', 'n/a')}") else: print("/stats deltas: unavailable") print("\nScenario results:") for result in results: marker = "PASS" if result.passed else "FAIL" print(f" [{marker}] {result.name:<24} {result.detail}") print("Direct Kafka fault injections: 30 additional (20 missing fields, 10 malformed JSON)") print("API rejects structural errors; direct injections independently verify worker handling.") passed = all(result.passed for result in results) print("\nOVERALL: " + ("PASS" if passed else "FAIL")) print("============================================================\n") return 0 if passed else 1 except Exception as exc: print(f"OVERALL: FAIL - verification could not complete: {exc}") return 1 finally: evidence.close() if __name__ == "__main__": parser = argparse.ArgumentParser( description="Assessment-focused Telecom CDR pipeline traffic generator and verifier" ) parser.add_argument( "--api-url", default="http://localhost:8000", help="Ingestion API base URL" ) parser.add_argument( "--clickhouse-url", default="http://localhost:8123", help="ClickHouse HTTP endpoint", ) parser.add_argument( "--stats-url", default=None, help="Aggregated /stats URL (default: /stats)", ) parser.add_argument( "--settle-timeout", type=float, default=15.0, help="Maximum seconds to wait for worker flush/DLQ processing", ) parser.add_argument( "--burst-workers", type=int, default=50, help="Concurrent client threads for the 500-record burst", ) parser.add_argument("--kafka-brokers", default=os.getenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092")) parser.add_argument("--raw-topic", default=os.getenv("KAFKA_RAW_TOPIC", "cdr.raw")) parser.add_argument("--dlq-topic", default=os.getenv("KAFKA_DLQ_TOPIC", "cdr.raw.dlq")) args = parser.parse_args() stats_url = args.stats_url or f"{args.api_url.rstrip('/')}/stats" sys.exit( run_verification( api_url=args.api_url, clickhouse_url=args.clickhouse_url, stats_url=stats_url, settle_timeout=args.settle_timeout, burst_workers=args.burst_workers, kafka_brokers=args.kafka_brokers, raw_topic=args.raw_topic, dlq_topic=args.dlq_topic, ) )