diff --git a/.env.sample b/.env.sample index e69de29..a4a3d92 100644 --- a/.env.sample +++ b/.env.sample @@ -0,0 +1,22 @@ +# ========================= +# Kafka / Redpanda +# ========================= + +REDPANDA_VERSION=v26.2.3 + +KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 + +# Raw CDR topic +KAFKA_RAW_TOPIC=cdr.raw +KAFKA_RAW_TOPIC_PARTITIONS=3 +KAFKA_RAW_TOPIC_REPLICATION_FACTOR=1 +KAFKA_RAW_TOPIC_RETENTION_MS=86400000 +KAFKA_RAW_TOPIC_COMPRESSION=lz4 + +# Dead Letter Queue +KAFKA_DLQ_TOPIC=cdr.raw.dlq +KAFKA_DLQ_TOPIC_PARTITIONS=1 +KAFKA_DLQ_TOPIC_REPLICATION_FACTOR=1 +KAFKA_DLQ_TOPIC_RETENTION_MS=604800000 + +KAFKA_CONSUMER_GROUP=cdr-processing-workers \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9ad126c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,96 @@ +name: telecom-cdr-pipeline + +services: + redpanda: + image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-v26.2.3} + + container_name: telecom-redpanda + + command: + - redpanda + - start + + # Kafka listeners: + # 9092 -> other Docker containers + # 19092 -> applications running on our host machine + - --kafka-addr=internal://0.0.0.0:9092,external://0.0.0.0:19092 + + # These are the addresses Redpanda tells Kafka clients to use. + - --advertise-kafka-addr=internal://redpanda:9092,external://localhost:19092 + + # Internal Redpanda node communication. + - --rpc-addr=redpanda:33145 + - --advertise-rpc-addr=redpanda:33145 + + # Suitable settings for local Docker development. + - --mode=dev-container + - --smp=1 + - --default-log-level=info + + ports: + - "19092:19092" + - "9644:9644" # Redpanda's Admin API + + volumes: + - redpanda-data:/var/lib/redpanda/data + + healthcheck: + test: + [ + "CMD", + "rpk", + "cluster", + "health", + "--exit-when-healthy" + ] + interval: 5s + timeout: 10s + retries: 10 + start_period: 10s + + redpanda-init: + image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-v26.2.3} + + depends_on: + redpanda: + condition: service_healthy + + environment: + KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS} + + KAFKA_RAW_TOPIC: ${KAFKA_RAW_TOPIC} + KAFKA_RAW_TOPIC_PARTITIONS: ${KAFKA_RAW_TOPIC_PARTITIONS} + KAFKA_RAW_TOPIC_REPLICATION_FACTOR: ${KAFKA_RAW_TOPIC_REPLICATION_FACTOR} + KAFKA_RAW_TOPIC_RETENTION_MS: ${KAFKA_RAW_TOPIC_RETENTION_MS} + KAFKA_RAW_TOPIC_COMPRESSION: ${KAFKA_RAW_TOPIC_COMPRESSION} + + KAFKA_DLQ_TOPIC: ${KAFKA_DLQ_TOPIC} + KAFKA_DLQ_TOPIC_PARTITIONS: ${KAFKA_DLQ_TOPIC_PARTITIONS} + KAFKA_DLQ_TOPIC_REPLICATION_FACTOR: ${KAFKA_DLQ_TOPIC_REPLICATION_FACTOR} + KAFKA_DLQ_TOPIC_RETENTION_MS: ${KAFKA_DLQ_TOPIC_RETENTION_MS} + + entrypoint: ["/bin/bash", "-c"] + + command: + - | + echo "Creating Kafka topics..." + + rpk topic create "$$KAFKA_RAW_TOPIC" \ + --if-not-exists \ + --brokers "$$KAFKA_BOOTSTRAP_SERVERS" \ + -p "$$KAFKA_RAW_TOPIC_PARTITIONS" \ + -r "$$KAFKA_RAW_TOPIC_REPLICATION_FACTOR" \ + -c retention.ms="$$KAFKA_RAW_TOPIC_RETENTION_MS" \ + -c compression.type="$$KAFKA_RAW_TOPIC_COMPRESSION" + + rpk topic create "$$KAFKA_DLQ_TOPIC" \ + --if-not-exists \ + --brokers "$$KAFKA_BOOTSTRAP_SERVERS" \ + -p "$$KAFKA_DLQ_TOPIC_PARTITIONS" \ + -r "$$KAFKA_DLQ_TOPIC_REPLICATION_FACTOR" \ + -c retention.ms="$$KAFKA_DLQ_TOPIC_RETENTION_MS" + + echo "Kafka topics initialized successfully." + +volumes: + redpanda-data: \ No newline at end of file