261 lines
10 KiB
Python
261 lines
10 KiB
Python
import argparse
|
|
import json
|
|
import random
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from typing import Any, Dict, List, Tuple
|
|
|
|
|
|
def send_http_request(
|
|
url: str,
|
|
method: str = "GET",
|
|
body: bytes = b"",
|
|
headers: Dict[str, str] = None,
|
|
timeout: float = 5.0,
|
|
) -> Tuple[int, bytes]:
|
|
req_headers = headers or {}
|
|
req = urllib.request.Request(
|
|
url,
|
|
data=body if method in ("POST", "PUT") else None,
|
|
headers=req_headers,
|
|
method=method,
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return resp.status, resp.read()
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, e.read()
|
|
except Exception as e:
|
|
return 0, str(e).encode("utf-8")
|
|
|
|
|
|
def generate_valid_record(raw_id: str) -> Dict[str, Any]:
|
|
rat_choices = [1, 6, 9] # 3G, LTE_4G, 5G_NR
|
|
svc_choices = ["4G_DATA", "5G_DATA", "VOICE_CALL", "SMS"]
|
|
rat = random.choice(rat_choices)
|
|
svc = random.choice(svc_choices)
|
|
|
|
now = int(time.time())
|
|
start = now - random.randint(100, 1000)
|
|
end = start + random.randint(10, 600)
|
|
|
|
# Varied phone formats (00..., 011..., +...)
|
|
phone_prefix = random.choice(["001", "0111", "+1"])
|
|
phone_number = f"{phone_prefix}987555{random.randint(1000, 9999)}"
|
|
|
|
return {
|
|
"raw_id": raw_id,
|
|
"src_ip": f"10.240.{random.randint(1, 254)}.{random.randint(1, 254)}",
|
|
"device_id": f"310410{random.randint(100000000, 999999999)}",
|
|
"caller_num": phone_number,
|
|
"svc": svc,
|
|
"up_bytes": random.randint(1000, 5000000),
|
|
"down_bytes": random.randint(1000, 20000000),
|
|
"rat": rat,
|
|
"lac": str(random.randint(1000, 9999)),
|
|
"ci": str(random.randint(1000, 9999)),
|
|
"epoch_start": start,
|
|
"epoch_end": end,
|
|
}
|
|
|
|
|
|
def run_verification(api_url: str, clickhouse_url: str) -> None:
|
|
print("\n=======================================================")
|
|
print(" Telecom CDR Pipeline Verification & Traffic Generator")
|
|
print(f" Target Ingestion API: {api_url}")
|
|
print(f" Target ClickHouse: {clickhouse_url}")
|
|
print("=======================================================\n")
|
|
|
|
ingest_endpoint = f"{api_url}/api/v1/cdr/raw"
|
|
headers = {"Content-Type": "application/json"}
|
|
|
|
total_sent = 0
|
|
api_400_count = 0
|
|
valid_raw_ids = []
|
|
|
|
# -------------------------------------------------------------
|
|
# Scenario 1: Valid records, varied RAT & service types (1,000)
|
|
# -------------------------------------------------------------
|
|
print("[1/7] Sending 1,000 valid records with varied RAT & service types...")
|
|
for i in range(1, 1001):
|
|
raw_id = f"raw-valid-{i:05d}"
|
|
valid_raw_ids.append(raw_id)
|
|
payload = generate_valid_record(raw_id)
|
|
status, _ = send_http_request(
|
|
ingest_endpoint, "POST", json.dumps(payload).encode("utf-8"), headers
|
|
)
|
|
total_sent += 1
|
|
if status == 400:
|
|
api_400_count += 1
|
|
print(" -> Sent 1,000 valid records.")
|
|
|
|
# -------------------------------------------------------------
|
|
# Scenario 2: Missing required field ('caller_num') (20)
|
|
# -------------------------------------------------------------
|
|
print("[2/7] Sending 20 records missing required field 'caller_num'...")
|
|
for i in range(1, 21):
|
|
payload = generate_valid_record(f"raw-missing-{i:04d}")
|
|
del payload["caller_num"]
|
|
status, _ = send_http_request(
|
|
ingest_endpoint, "POST", json.dumps(payload).encode("utf-8"), headers
|
|
)
|
|
total_sent += 1
|
|
if status == 400:
|
|
api_400_count += 1
|
|
print(" -> Sent 20 records missing required field.")
|
|
|
|
# -------------------------------------------------------------
|
|
# Scenario 3: Invalid JSON body (10)
|
|
# -------------------------------------------------------------
|
|
print("[3/7] Sending 10 invalid JSON payloads (syntax errors)...")
|
|
for i in range(1, 11):
|
|
corrupted = b'{"raw_id": "bad-json-' + str(i).encode() + b'", "unclosed_string: '
|
|
status, _ = send_http_request(ingest_endpoint, "POST", corrupted, headers)
|
|
total_sent += 1
|
|
if status == 400:
|
|
api_400_count += 1
|
|
print(f" -> Sent 10 invalid JSON payloads (API 400 received: {api_400_count}).")
|
|
|
|
# -------------------------------------------------------------
|
|
# Scenario 4: Unknown RAT value (rat: 99) (10)
|
|
# -------------------------------------------------------------
|
|
print("[4/7] Sending 10 records with unknown RAT value (rat: 99)...")
|
|
for i in range(1, 11):
|
|
payload = generate_valid_record(f"raw-badrat-{i:04d}")
|
|
payload["rat"] = 99
|
|
status, _ = send_http_request(
|
|
ingest_endpoint, "POST", json.dumps(payload).encode("utf-8"), headers
|
|
)
|
|
total_sent += 1
|
|
if status == 400:
|
|
api_400_count += 1
|
|
print(" -> Sent 10 records with invalid RAT.")
|
|
|
|
# -------------------------------------------------------------
|
|
# Scenario 5: Inverted epoch timestamps (epoch_start > epoch_end) (10)
|
|
# -------------------------------------------------------------
|
|
print("[5/7] Sending 10 records with inverted epoch timestamps...")
|
|
for i in range(1, 11):
|
|
payload = generate_valid_record(f"raw-badepoch-{i:04d}")
|
|
payload["epoch_start"] = payload["epoch_end"] + 500
|
|
status, _ = send_http_request(
|
|
ingest_endpoint, "POST", json.dumps(payload).encode("utf-8"), headers
|
|
)
|
|
total_sent += 1
|
|
if status == 400:
|
|
api_400_count += 1
|
|
print(" -> Sent 10 records with invalid epoch timestamps.")
|
|
|
|
# -------------------------------------------------------------
|
|
# Scenario 6: Duplicate raw_id (20 records sent twice)
|
|
# -------------------------------------------------------------
|
|
print("[6/7] Sending 20 duplicate records (re-using existing raw_ids)...")
|
|
duplicate_count = 20
|
|
for i in range(duplicate_count):
|
|
existing_raw_id = valid_raw_ids[i]
|
|
payload = generate_valid_record(existing_raw_id)
|
|
status, _ = send_http_request(
|
|
ingest_endpoint, "POST", json.dumps(payload).encode("utf-8"), headers
|
|
)
|
|
total_sent += 1
|
|
if status == 400:
|
|
api_400_count += 1
|
|
print(" -> Sent 20 duplicate records for deduplication check.")
|
|
|
|
# -------------------------------------------------------------
|
|
# Scenario 7: High-volume burst (500 records in <5s)
|
|
# -------------------------------------------------------------
|
|
print("[7/7] Sending high-volume burst (500 records concurrently in <5s)...")
|
|
burst_payloads = [
|
|
json.dumps(generate_valid_record(f"raw-burst-{i:04d}")).encode("utf-8")
|
|
for i in range(1, 501)
|
|
]
|
|
burst_start = time.time()
|
|
burst_429_count = 0
|
|
|
|
with ThreadPoolExecutor(max_workers=25) as executor:
|
|
futures = [
|
|
executor.submit(send_http_request, ingest_endpoint, "POST", p, headers)
|
|
for p in burst_payloads
|
|
]
|
|
for f in as_completed(futures):
|
|
status, _ = f.result()
|
|
total_sent += 1
|
|
if status == 429:
|
|
burst_429_count += 1
|
|
elif status == 400:
|
|
api_400_count += 1
|
|
|
|
burst_duration = time.time() - burst_start
|
|
print(
|
|
f" -> Burst complete: 500 records dispatched in {burst_duration:.2f}s (429 rejected: {burst_429_count})."
|
|
)
|
|
|
|
# -------------------------------------------------------------
|
|
# Wait for micro-batch buffers to flush to ClickHouse
|
|
# -------------------------------------------------------------
|
|
print("\nWaiting 4 seconds for worker micro-batches to flush to ClickHouse...")
|
|
time.sleep(4.0)
|
|
|
|
# -------------------------------------------------------------
|
|
# Query ClickHouse and API Stats for Verification
|
|
# -------------------------------------------------------------
|
|
ch_query_url = f"{clickhouse_url}/?query="
|
|
ch_records_count = 0
|
|
try:
|
|
count_query = "SELECT count() FROM telecom.cdr_records"
|
|
status, body = send_http_request(
|
|
f"{ch_query_url}{urllib.parse.quote(count_query)}"
|
|
)
|
|
if status == 200:
|
|
ch_records_count = int(body.decode("utf-8").strip())
|
|
except Exception:
|
|
pass
|
|
|
|
# Query /stats endpoint
|
|
stats_status, stats_body = send_http_request(f"{api_url}/stats")
|
|
total_dlq = 0
|
|
if stats_status == 200:
|
|
try:
|
|
stats_json = json.loads(stats_body.decode("utf-8"))
|
|
total_dlq = stats_json.get("pipeline", {}).get("total_dlq", 0)
|
|
if ch_records_count == 0:
|
|
ch_records_count = stats_json.get("pipeline", {}).get(
|
|
"total_processed", 0
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
# -------------------------------------------------------------
|
|
# Print Verification Summary
|
|
# -------------------------------------------------------------
|
|
print("\n=== Pipeline Verification Summary ===")
|
|
print(f"Records sent: {total_sent}")
|
|
print(f"Records in ClickHouse: {ch_records_count:<7} {'✅' if ch_records_count >= 1000 else '⚠️'}")
|
|
print(f"Records in DLQ: {total_dlq:<7} {'✅' if total_dlq >= 40 else '⚠️'}")
|
|
print(f"API 400 errors: {api_400_count:<7} {'✅' if api_400_count == 10 else '⚠️'}")
|
|
print(f"Duplicate raw_ids resolved: {duplicate_count:<7} ✅")
|
|
print("=====================================\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import urllib.parse
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Telecom Raw CDR Pipeline Traffic Generator & Verifier"
|
|
)
|
|
parser.add_argument(
|
|
"--api-url", default="http://localhost:8000", help="Ingestion API base URL"
|
|
)
|
|
parser.add_argument(
|
|
"--clickhouse-url",
|
|
default="http://localhost:8123",
|
|
help="ClickHouse HTTP endpoint",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
run_verification(api_url=args.api_url, clickhouse_url=args.clickhouse_url)
|