22
33# Copyright (c) 2026 Relax Authors. All Rights Reserved.
44#
5- # Entrypoint for Ray Job tasks.
5+ # Entrypoint / source helper for Ray Job tasks.
66# The Ray cluster is already running. This script MUST NOT kill ray or stop the
7- # cluster. It only cleans up residual python/sglang processes and then runs the
8- # training script directly via python3 .
7+ # cluster. It only cleans up residual python/sglang processes and then sets up
8+ # the environment for running training against an existing Ray cluster .
99#
10- # Usage:
11- # bash scripts/entrypoint/ray-job.sh <run-script> [extra-args...]
10+ # Two usage modes:
11+ # 1) Entry-point mode — first argument is a .sh script path:
12+ # bash scripts/entrypoint/ray-job.sh <run-script> [extra-args...]
13+ # Sets up env, cleans residual processes, then execs the run script.
1214#
13- # Example:
14- # bash scripts/entrypoint/ray-job.sh scripts/training/text/run-qwen35-9B-8xgpu-async.sh
15- # bash scripts/entrypoint/ray-job.sh scripts/training/text/run-qwen35-9B-8xgpu-async.sh --lr 5e-7
15+ # Example:
16+ # bash scripts/entrypoint/ray-job.sh scripts/training/text/run-qwen35-9B-8xgpu-async.sh
17+ # bash scripts/entrypoint/ray-job.sh scripts/training/text/run-qwen35-9B-8xgpu-async.sh --lr 5e-7
18+ #
19+ # 2) Source mode — no .sh script arg (like local.sh):
20+ # source scripts/entrypoint/ray-job.sh
21+ # Sets up env only, so the caller can continue execution.
1622#
1723# Environment variables (optional):
1824# MEGATRON - Path to Megatron-LM (default: /root/Megatron-LM/)
1925# RELAX - Path to Relax project (default: ../../)
2026
21- set -eo pipefail
22-
23- DIR=" $( cd " $( dirname " ${BASH_SOURCE[0]} " ) " > /dev/null 2>&1 && pwd) "
24-
25- # ── argument parsing ────────────────────────────────────────────────────────
26- RUN_SCRIPT=" ${1:- } "
27- if [ -z " $RUN_SCRIPT " ]; then
28- echo " Usage: $0 <run-script> [extra-args...]" >&2
29- exit 1
27+ # Guard: skip if already sourced by another entrypoint
28+ if [ -n " ${RELAX_ENTRYPOINT_MODE:- } " ]; then
29+ return 0 2> /dev/null || exit 0
3030fi
31- shift # remaining args are extra overrides
3231
33- if [ ! -f " $RUN_SCRIPT " ]; then
34- echo " ERROR: run script not found: $RUN_SCRIPT " >&2
35- exit 1
32+ # ── mode detection ──────────────────────────────────────────────────────────
33+ # Entry-point mode: directly executed AND first arg is an existing .sh file.
34+ # Otherwise act as a sourced setup script.
35+ _RAY_JOB_RUN_SCRIPT=" "
36+ if [[ " ${BASH_SOURCE[0]} " == " ${0} " ]]; then
37+ _RAY_JOB_FIRST_ARG=" ${1:- } "
38+ if [ -n " $_RAY_JOB_FIRST_ARG " ] && [ -f " $_RAY_JOB_FIRST_ARG " ] && [[ " $_RAY_JOB_FIRST_ARG " == * .sh ]]; then
39+ _RAY_JOB_RUN_SCRIPT=" $_RAY_JOB_FIRST_ARG "
40+ shift
41+ else
42+ echo " Usage: $0 <run-script.sh> [extra-args...]" >&2
43+ exit 1
44+ fi
3645fi
3746
47+ set -eo pipefail
48+
49+ DIR=" $( cd " $( dirname " ${BASH_SOURCE[0]} " ) " > /dev/null 2>&1 && pwd) "
50+
3851# ── clean up residual python/sglang processes (NOT ray) ─────────────────────
3952# IMPORTANT: Do NOT pkill ray or run ray stop — the cluster is managed externally.
4053echo " === Cleaning up residual python/sglang processes ==="
@@ -46,14 +59,13 @@ ray job list | grep RUNNING | grep -v job_id=None | grep -oP "submission_id='\\K
4659set -x
4760
4861# ── environment setup ───────────────────────────────────────────────────────
49- # Use the first GPU node as MASTER_ADDR and SLIME_HOST_IP (prefer head node)
62+ # Use the first GPU node as MASTER_ADDR (prefer head node)
5063export MASTER_ADDR=$( ray list nodes --format json | jq -r '
5164 map(select(.state == "ALIVE" and (.resources_total.GPU // 0) > 0)) |
5265 sort_by(.is_head_node | not) |
5366 .[0].node_ip
5467' )
5568
56- DIR=" $( cd " $( dirname " ${BASH_SOURCE[0]} " ) " > /dev/null 2>&1 && pwd) "
5769export PYTHONUNBUFFERED=1
5870export CUDA_DEVICE_MAX_CONNECTIONS=1
5971export MEGATRON=${MEGATRON:-/ root/ Megatron-LM/ }
7486fi
7587echo " HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)"
7688
77- # ── delegate to run script ──────────────────────────────────────────────────
78- echo " === Launching training script: $RUN_SCRIPT ==="
89+ # ── entrypoint mode & runtime env ──────────────────────────────────────────
7990export RELAX_ENTRYPOINT_MODE=" ray-job"
8091RAY_DEBUG=${RAY_DEBUG:- " 0" }
8192RAY_DEBUG_POST_MORTEM=${RAY_DEBUG_POST_MORTEM:- " 0" }
8293
83- # Runtime env for ray-job mode (empty, env inherited from Ray cluster)
94+ # Runtime env for ray-job mode (env inherited from Ray cluster)
8495export RUNTIME_ENV_JSON=" {
8596\" env_vars\" : {
8697 \" PYTHONUNBUFFERED\" : \" 1\" ,
@@ -93,4 +104,11 @@ export RUNTIME_ENV_JSON="{
93104 \" RAY_DEBUG_POST_MORTEM\" : \" ${RAY_DEBUG_POST_MORTEM} \"
94105}
95106}"
96- exec bash " $RUN_SCRIPT " " $@ "
107+
108+ echo " === Ray-job environment ready ==="
109+
110+ # ── delegate to run script (entry-point mode only) ─────────────────────────
111+ if [ -n " $_RAY_JOB_RUN_SCRIPT " ]; then
112+ echo " === Launching training script: $_RAY_JOB_RUN_SCRIPT ==="
113+ exec bash " $_RAY_JOB_RUN_SCRIPT " " $@ "
114+ fi
0 commit comments