.env.example

# Kafka Configuration
KAFKA_BOOTSTRAP_SERVERS=your-kafka-server:9092

# Kafka Authentication Method: SASL or SSL
# SASL - Use API Key/Secret (Confluent Cloud default)
# SSL  - Use Certificate-based authentication
KAFKA_AUTH_METHOD=SASL

# SASL Authentication (API Key/Secret) - Required if KAFKA_AUTH_METHOD=SASL
KAFKA_API_KEY=your-api-key
KAFKA_API_SECRET=your-api-secret

# SSL Certificate Authentication - Required if KAFKA_AUTH_METHOD=SSL
# Uncomment and set paths to your certificate files
# KAFKA_SSL_CA_LOCATION=/path/to/ca-cert.pem
# KAFKA_SSL_CERTIFICATE_LOCATION=/path/to/client-cert.pem
# KAFKA_SSL_KEY_LOCATION=/path/to/client-key.pem
# KAFKA_SSL_KEY_PASSWORD=your-key-password-if-encrypted
# KAFKA_SSL_CHECK_HOSTNAME=true
# Set to false to disable hostname verification (ssl.endpoint.identification.algorithm=none)

# Kafka Consumer Settings
KAFKA_CONSUMER_GROUP=flink-sigma
KAFKA_STARTING_OFFSET=latest

# Flink Configuration
# FLINK_PARALLELISM controls matcher and sink parallelism
FLINK_PARALLELISM=4

# OPTIONAL: Override source parallelism if Kafka topic has fewer partitions
# Only set this if your Kafka input topic has fewer partitions than FLINK_PARALLELISM
# Example: Topic has 2 partitions but you want FLINK_PARALLELISM=8
#   → Set KAFKA_SOURCE_PARALLELISM=2 (to match partition count)
#   → Matcher and sink will still use 8 (from FLINK_PARALLELISM)
# KAFKA_SOURCE_PARALLELISM=2

FLINK_CHECKPOINT_INTERVAL_MS=60000
FLINK_CHECKPOINT_TIMEOUT_MS=300000
FLINK_MIN_PAUSE_BETWEEN_CHECKPOINTS_MS=30000

# Docker Resource Limits (OPTIONAL - recommended for production)
# Purpose: Safety net to prevent resource exhaustion in case of bugs/leaks
# If not set, containers can use all available host resources
# Set these if:
#   - Running multiple services on same machine
#   - Want protection against memory leaks in Python UDFs
#   - Production environment with strict resource allocation
# Leave unset for:
#   - Development on dedicated machine
#   - Single-service deployment
#   - Maximum flexibility

# Event Keying Strategy for Horizontal Scaling
# Options:
#   - hash: Hash-based distribution (recommended, fast, no JSON parsing)
#   - computer: Key by Computer/hostname (legacy, requires JSON parsing)
#   - round_robin: Simple round-robin (legacy, testing only)
KEYING_STRATEGY=hash

# Key Groups Per Task (affects performance vs load balancing trade-off)
# Total key buckets = FLINK_PARALLELISM × KEY_GROUPS_PER_TASK
# Each key group triggers separate on_timer() with rule matching overhead
#
# Trade-off examples:
#   Parallelism=10, KeyGroups=1  → 10 buckets  (minimal overhead, uneven load on scale)
#   Parallelism=10, KeyGroups=3  → 30 buckets  (balanced, recommended)
#   Parallelism=10, KeyGroups=10 → 100 buckets (even load, high overhead)
#
# Recommended: 2-5 for production
KEY_GROUPS_PER_TASK=3

# Maximum Parallelism (for autoscaling environments)
# Should match pipeline.max-parallelism in Kubernetes deployment (720)
# Used to calculate bucket count when autoscaler changes parallelism
# Example: max=720 → ~180 buckets (supports parallelism up to 60)
MAX_PARALLELISM=720

# State Backend Configuration
# Options:
#   - memory: Heap-based (fast, but limited by JVM heap size)
#   - rocksdb: Disk-based (supports large state, incremental checkpoints) - RECOMMENDED
STATE_BACKEND=rocksdb

# Checkpoint Storage Path
# Options:
#   - file:///opt/flink/checkpoints (default, Docker volume)
#   - file:///mnt/nfs/flink-checkpoints (production NFS mount)
# For production: mount NFS and update this path
CHECKPOINT_PATH=file:///opt/flink/checkpoints

# Production NFS Setup:
# 1. Mount NFS on host: /mnt/nfs/flink-checkpoints
# 2. Update docker-compose.yml volume: /mnt/nfs/flink-checkpoints:/mnt/nfs/flink-checkpoints
# 3. Set CHECKPOINT_PATH=file:///mnt/nfs/flink-checkpoints
# See DEPLOYMENT.md for complete guide

# RocksDB Performance Tuning (only used when STATE_BACKEND=rocksdb)
# Write Buffer: Size before flushing to disk
# Larger values = fewer flushes, better write throughput, more memory
# Recommendation: 64-128MB for production
ROCKSDB_WRITEBUFFER_SIZE_MB=64

# Number of write buffers (total memory = size * count)
# Recommendation: 3-4 for production
ROCKSDB_WRITEBUFFER_COUNT=3

# Block Cache: Cache for frequently accessed data
# Larger values = better read performance, more memory
# Recommendation: 256-512MB for production
ROCKSDB_BLOCK_CACHE_SIZE_MB=256

# Block Size: Size of data blocks in cache
# Smaller = better for point lookups, larger = better for scans
# Recommendation: 4-16KB for streaming workloads
ROCKSDB_BLOCK_SIZE_KB=4

# Compaction Style:
#   - LEVEL: Better read performance, less space amplification (recommended)
#   - UNIVERSAL: Better write performance, more space amplification
#   - FIFO: Best write performance, no compaction (only for TTL use cases)
ROCKSDB_COMPACTION_STYLE=LEVEL

# Number of background threads for compaction/flush
# Recommendation: 4-8 threads for production
ROCKSDB_THREAD_NUM=4

# Watermark Configuration (Event-Time Processing)
# Enable watermarks for handling out-of-order events and late arrivals
# Options:
#   - false: Processing-time (lower latency, simpler) - RECOMMENDED for real-time detection
#   - true: Event-time (handles late events, more complex)
ENABLE_WATERMARKS=false

# Maximum out-of-orderness for watermarks (in seconds)
# How late events can arrive before being considered too late
# Only used when ENABLE_WATERMARKS=true
# Larger values = more tolerance for late events, higher latency
WATERMARK_OUT_OF_ORDERNESS_SECONDS=10

# Idle timeout for watermark sources (in seconds)
# Prevents watermark stalling when some partitions have no data
# Only used when ENABLE_WATERMARKS=true
WATERMARK_IDLE_TIMEOUT_SECONDS=60

WINDOW_SIZE_SECONDS=30

# Max buffered events per key before forcing an early window flush.
# Helps prevent oversized gRPC payloads between JVM and Python operators.
WINDOW_COUNT_THRESHOLD=100000

# Logging Configuration
LOG_LEVEL=INFO

# Log format for Python application logs:
#   - json: structlog JSON format (production, default)
#   - console: structlog colored console format (development)
#
# Java Flink logs configuration:
#   - Default: log4j2.properties (JSON format, INFO level)
#   - For debugging: Copy log4j2-console.properties to log4j2.properties
#     (console format, ERROR level only)
#
LOG_FORMAT=json