docker-compose.yml

# Main Docker Compose file.
#
# It defines all the services:
# - Kafka (message broker with KRaft mode)
# - PostgreSQL (I/O target)
# - Prometheus (metrics collection)
# - Grafana (dashboards for live monitoring)
# - cAdvisor (container-level CPU/memory metrics)
# - Consumer (Node.js application being benchmarked)
# - k6 (load generator)
# - Python & Node (analysis and dev utilities)

services:
  # --- Kafka ---
  # Apache Kafka in KRaft mode (built-in consensus, without Zookeeper).
  kafka:
    image: apache/kafka:3.9.0
    container_name: kafka
    environment:
      # KRaft mode configuration (Kafka manages its own metadata)
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      # Three listeners: internal (9092), controller (9093), external/host (9094)
      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      # Replication factor 1 (single broker, no redundancy)
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      # Short retention (don't need to keep old messages)
      KAFKA_LOG_RETENTION_HOURS: 1
      CLUSTER_ID: "benchmark-kafka-cluster-001"
    ports:
      # Only the external listener is mapped to the host (for k6 and debugging)
      - "9094:9094"
    healthcheck:
      test: /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 || exit 1
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 30s
    volumes:
      - kafka-data:/tmp/kraft-combined-logs

  # One-shot container that creates the topic (idempotent).
  kafka-init:
    image: apache/kafka:3.9.0
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      KAFKA_TOPIC: ${KAFKA_TOPIC:-enope}
    entrypoint:
      [
        "bash",
        "-c",
        '/opt/kafka/bin/kafka-topics.sh --create --if-not-exists --bootstrap-server kafka:9092 --topic "$$KAFKA_TOPIC" --partitions 8 --replication-factor 1 && echo "Topic $$KAFKA_TOPIC created successfully (8 partitions)"',
      ]
    restart: "no"

  # --- PostgreSQL ---
  # Used by I/O and Mixed workloads to INSERT processed messages.
  postgres:
    image: postgres:17.10-alpine
    container_name: postgres
    command: ["postgres", "-c", "max_connections=200"]
    environment:
      POSTGRES_DB: ${POSTGRES_DB:-benchmark}
      POSTGRES_USER: ${POSTGRES_USER:-benchmark}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-benchmark}
    ports:
      - "5432:5432" # host-side access for psql/debugging
    healthcheck:
      test: pg_isready -U "${POSTGRES_USER:-benchmark}" -d "${POSTGRES_DB:-benchmark}"
      interval: 5s
      timeout: 5s
      retries: 5
    volumes:
      - postgres-data:/var/lib/postgresql/data

  # --- Prometheus ---
  # Collects metrics from the consumer and cAdvisor.
  prometheus:
    image: prom/prometheus:v3.4.0
    container_name: prometheus
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.retention.time=7d"
    ports:
      - "9091:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus-data:/prometheus

  # --- Grafana ---
  # Dashboards for live monitoring.
  grafana:
    image: grafana/grafana:11.6.0
    container_name: grafana
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_USERS_ALLOW_SIGN_UP: "false"
    ports:
      - "3000:3000"
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
      - grafana-data:/var/lib/grafana
    depends_on:
      - prometheus

  # --- cAdvisor (macOS) ---
  # Collects container-level CPU and memory metrics (macOS setup).
  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.55.1
    container_name: cadvisor
    ports:
      - "8080:8080"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /run/containerd/containerd.sock:/run/containerd/containerd.sock:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
    privileged: true
    cgroup: host
    profiles:
      - monitoring

  # --- cAdvisor (Linux) ---
  # Collects container-level CPU and memory metrics (Linux setup).
  cadvisor-linux:
    image: gcr.io/cadvisor/cadvisor:v0.55.1
    container_name: cadvisor
    ports:
      - "8080:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /run/containerd/containerd.sock:/run/containerd/containerd.sock:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
    privileged: true
    cgroup: host
    profiles:
      - monitoring-linux

  # --- Node.js consumer ---
  # Each consumer will have:
  # - 1.5 CPU cores (1 core for the single-threaded event loop + 0.5 headroom for V8 GC/JIT background threads and the libuv thread pool)
  # - 1GB RAM (swap disabled)
  # - V8 heap auto-sized by Node.js from the cgroup limit
  # - 256 PIDs max (safety limit against fork bombs)
  consumer:
    build:
      context: ./consumer
      dockerfile: Dockerfile
    deploy:
      resources:
        limits:
          cpus: "1.5"
          memory: 1G
          pids: 256
        reservations:
          cpus: "0.5"
          memory: 128M
    memswap_limit: 1G # Value equal to memory disables swap
    environment:
      KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:9092}
      KAFKA_TOPIC: ${KAFKA_TOPIC:-enope}
      GROUP_ID: ${GROUP_ID:-benchmark-group}
      WORKLOAD_TYPE: ${WORKLOAD_TYPE:-io} # Override with `docker-compose.{io,cpu,mix}.yml`
      METRICS_PORT: ${METRICS_PORT:-9090}
      POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
      POSTGRES_PORT: ${POSTGRES_PORT:-5432}
      POSTGRES_DB: ${POSTGRES_DB:-benchmark}
      POSTGRES_USER: ${POSTGRES_USER:-benchmark}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-benchmark}
    expose:
      - "${METRICS_PORT:-9090}"
    healthcheck:
      test:
        [
          "CMD",
          "node",
          "-e",
          "fetch('http://localhost:' + (process.env.METRICS_PORT || '9090') + '/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))",
        ]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 15s
    depends_on:
      kafka-init:
        condition: service_completed_successfully
      postgres:
        condition: service_healthy

  # --- k6 + xk6-kafka ---
  # Produces messages to Kafka at configurable rates.
  k6:
    build:
      context: ./k6
      dockerfile: Dockerfile
    volumes:
      - ./k6:/scripts:ro
    environment:
      KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:9092}
      KAFKA_TOPIC: ${KAFKA_TOPIC:-enope}
    depends_on:
      kafka-init:
        condition: service_completed_successfully
    profiles:
      - load

  # --- Python analysis scripts ---
  python:
    build:
      context: .
      dockerfile: scripts/Dockerfile
    working_dir: /workspace
    volumes:
      - ./scripts:/workspace/scripts
      - ./results:/workspace/results
      - ./ruff.toml:/workspace/ruff.toml:ro
    profiles:
      - tools

  # --- Node.js npm scripts ---
  node-tools:
    image: node:24.17.0-slim
    working_dir: /workspace
    volumes:
      - .:/workspace
    profiles:
      - tools
    entrypoint: ["bash", "-c"]

# Volumes for data persistence and isolation.
volumes:
  kafka-data:
  postgres-data:
  prometheus-data:
  grafana-data: