docker-compose.sglang.yml

# SGLang server — plain serving, and the rollout/weight-sync pairing for env-GRPO. The default image
# is the NCCL-aligned Dockerfile.sglang build: upstream's NCCL does not match uv.lock's pin, and a
# skew hangs the sync group. Host networking; SGLANG_CUDA_DEVICES must not overlap the trainer's.
#   SGLANG_MODEL=<hub id|local dir> docker compose -f docker-compose.sglang.yml up   # `down` needs it too
#   (add -f docker-compose.sglang.efa.yml on an EFA host)

services:
  sglang-server:
    image: ${SGLANG_IMAGE:-sglang-server:0.5.17}
    ipc: host
    network_mode: host
    environment:
      - CUDA_VISIBLE_DEVICES=${SGLANG_CUDA_DEVICES:-7}
      - HF_HOME=${HF_HOME:-/root/.cache/huggingface}
      - HF_TOKEN=${HF_TOKEN:-}
      # Weight sync shares an NCCL group with the trainer in another container. cuMem must match on
      # both ends: SGLang sets NCCL_CUMEM_ENABLE=0 process-wide unless it is already set, the
      # trainer's NCCL has it on, and the mismatch fails the first cross-container buffer import
      # (`ncclP2pImportShareableBuffer ... invalid argument`). With parity the same-host group takes
      # CUDA IPC over NVLink.
      - NCCL_CUMEM_ENABLE=${NCCL_CUMEM_ENABLE:-1}
      # No-fabric defaults (InfiniBand off, socket net); the EFA overlay overrides both.
      - NCCL_NET=${NCCL_NET:-Socket}
      - NCCL_IB_DISABLE=${NCCL_IB_DISABLE:-1}
      # Keep the socket transport off Docker's bridge and per-container veth pairs.
      - NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-^docker,veth}
      # Passed through bare from the shell or .env (unset stays unset): a protocol table set on the
      # trainer alone hangs the first collective, so a trainer that pins NCCL_PROTO needs the server
      # pinned the same way, on every recipe.
      - NCCL_PROTO
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    volumes:
      # Mounted at the same path inside the container, so HF_HOME resolves either side.
      - ${HF_HOME:-~/.cache/huggingface}:${HF_HOME:-/root/.cache/huggingface}
      # Only needed when SGLANG_MODEL is a local path rather than a hub id; must contain it.
      - ${SGLANG_MODEL_DIR:-/mnt/models}:${SGLANG_MODEL_DIR:-/mnt/models}:ro
    # Two flags are required and silent when wrong: --tool-call-parser (tool-using environments need
    # the calls parsed; `auto` reads the parser off the chat template) and --moe-runner-backend
    # triton (the auto runners bypass R3 capture and repack expert weights). --attention-backend is a
    # per-family need, --trust-remote-code a per-checkpoint one, and SGLANG_EXTRA_ARGS is appended
    # last so a flag repeated there wins. Values per family: agent-docs/infrastructure/rollout-servers.md
    command: >
      python3 -m sglang.launch_server
      --model-path ${SGLANG_MODEL:?set SGLANG_MODEL to a hub id or a local checkpoint dir}
      --host 0.0.0.0
      --port ${SGLANG_PORT:-30000}
      --tp-size ${SGLANG_TP:-1}
      --mem-fraction-static ${SGLANG_GPU_MEM:-0.85}
      --tool-call-parser ${SGLANG_TOOL_PARSER:-auto}
      --moe-runner-backend ${SGLANG_MOE_RUNNER_BACKEND:-triton}
      ${SGLANG_ENABLE_R3:+--enable-return-routed-experts}
      ${SGLANG_REASONING_PARSER:+--reasoning-parser ${SGLANG_REASONING_PARSER}}
      ${SGLANG_CHAT_TEMPLATE:+--chat-template ${SGLANG_CHAT_TEMPLATE}}
      ${SGLANG_ATTENTION_BACKEND:+--attention-backend ${SGLANG_ATTENTION_BACKEND}}
      ${SGLANG_TRUST_REMOTE_CODE:+--trust-remote-code}
      ${SGLANG_EXTRA_ARGS:-}
    healthcheck:
      test: ["CMD-SHELL", "python3 -c \"import urllib.request; urllib.request.urlopen('http://localhost:${SGLANG_PORT:-30000}/health')\""]
      interval: 15s
      timeout: 10s
      retries: 40
      start_period: 180s
    restart: unless-stopped