Skip to content

Commit c3f66e9

Browse files
committed
refactor(entrypoint): dual-mode ray-job.sh + auto-delegate
# ♻️ Refactor ## Make ray-job.sh support both entry-point and source modes - Detect mode via BASH_SOURCE vs $0 plus first-arg .sh file check, so sourced callers cannot accidentally trigger entry-point exec from leaked positional args - Entry-point mode: set up env then exec the run script (unchanged CLI) - Source mode: set up env only and fall through to the caller - Drop duplicate DIR= assignment and unused SLIME_HOST_IP comment - Add RELAX_ENTRYPOINT_MODE guard to match local.sh ## Auto-delegate local.sh to ray-job.sh inside an existing cluster - At the top of local.sh, when RAY_ADDRESS is set and `ray status` succeeds, source ray-job.sh instead of tearing down and restarting a local Ray head node - Preserves existing single-node flow otherwise
1 parent a94226f commit c3f66e9

2 files changed

Lines changed: 62 additions & 28 deletions

File tree

‎scripts/entrypoint/local.sh‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
# It is designed to be *sourced* by run-*.sh scripts when no external entrypoint
88
# (spmd-multinode.sh or ray-job.sh) has been used.
99
#
10+
# When an existing Ray cluster is detected (RAY_ADDRESS set and `ray status` OK),
11+
# this script delegates to `ray-job.sh` (source mode) instead of starting a new
12+
# local Ray head node.
13+
#
1014
# Usage (from a run script):
1115
# source scripts/entrypoint/local.sh
1216
#
@@ -22,6 +26,19 @@ if [ -n "${RELAX_ENTRYPOINT_MODE:-}" ]; then
2226
return 0 2>/dev/null || exit 0
2327
fi
2428

29+
_LOCAL_SH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
30+
31+
# ── delegate to ray-job.sh when inside an existing Ray cluster ─────────────
32+
# When RAY_ADDRESS is set AND `ray status` succeeds, we're already part of an
33+
# externally-managed Ray cluster. Skip local Ray startup / process cleanup and
34+
# fall through to ray-job.sh (source mode) for env setup.
35+
if [ -n "${RAY_ADDRESS:-}" ] && timeout 5 ray status >/dev/null 2>&1; then
36+
echo "=== Detected existing Ray cluster (RAY_ADDRESS=$RAY_ADDRESS); delegating to ray-job.sh ==="
37+
# shellcheck source=./ray-job.sh
38+
source "${_LOCAL_SH_DIR}/ray-job.sh"
39+
return 0 2>/dev/null || exit 0
40+
fi
41+
2542
set -eo pipefail
2643

2744
# ── process cleanup ─────────────────────────────────────────────────────────
@@ -38,7 +55,6 @@ pkill -9 python 2>/dev/null || true
3855
set -x
3956

4057
# ── environment setup ───────────────────────────────────────────────────────
41-
_LOCAL_SH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
4258
unset MASTER_ADDR 2>/dev/null || true
4359
export PYTHONUNBUFFERED=1
4460
export CUDA_DEVICE_MAX_CONNECTIONS=1

‎scripts/entrypoint/ray-job.sh‎

Lines changed: 45 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,52 @@
22

33
# Copyright (c) 2026 Relax Authors. All Rights Reserved.
44
#
5-
# Entrypoint for Ray Job tasks.
5+
# Entrypoint / source helper for Ray Job tasks.
66
# The Ray cluster is already running. This script MUST NOT kill ray or stop the
7-
# cluster. It only cleans up residual python/sglang processes and then runs the
8-
# training script directly via python3.
7+
# cluster. It only cleans up residual python/sglang processes and then sets up
8+
# the environment for running training against an existing Ray cluster.
99
#
10-
# Usage:
11-
# bash scripts/entrypoint/ray-job.sh <run-script> [extra-args...]
10+
# Two usage modes:
11+
# 1) Entry-point mode — first argument is a .sh script path:
12+
# bash scripts/entrypoint/ray-job.sh <run-script> [extra-args...]
13+
# Sets up env, cleans residual processes, then execs the run script.
1214
#
13-
# Example:
14-
# bash scripts/entrypoint/ray-job.sh scripts/training/text/run-qwen35-9B-8xgpu-async.sh
15-
# bash scripts/entrypoint/ray-job.sh scripts/training/text/run-qwen35-9B-8xgpu-async.sh --lr 5e-7
15+
# Example:
16+
# bash scripts/entrypoint/ray-job.sh scripts/training/text/run-qwen35-9B-8xgpu-async.sh
17+
# bash scripts/entrypoint/ray-job.sh scripts/training/text/run-qwen35-9B-8xgpu-async.sh --lr 5e-7
18+
#
19+
# 2) Source mode — no .sh script arg (like local.sh):
20+
# source scripts/entrypoint/ray-job.sh
21+
# Sets up env only, so the caller can continue execution.
1622
#
1723
# Environment variables (optional):
1824
# MEGATRON - Path to Megatron-LM (default: /root/Megatron-LM/)
1925
# RELAX - Path to Relax project (default: ../../)
2026

21-
set -eo pipefail
22-
23-
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
24-
25-
# ── argument parsing ────────────────────────────────────────────────────────
26-
RUN_SCRIPT="${1:-}"
27-
if [ -z "$RUN_SCRIPT" ]; then
28-
echo "Usage: $0 <run-script> [extra-args...]" >&2
29-
exit 1
27+
# Guard: skip if already sourced by another entrypoint
28+
if [ -n "${RELAX_ENTRYPOINT_MODE:-}" ]; then
29+
return 0 2>/dev/null || exit 0
3030
fi
31-
shift # remaining args are extra overrides
3231

33-
if [ ! -f "$RUN_SCRIPT" ]; then
34-
echo "ERROR: run script not found: $RUN_SCRIPT" >&2
35-
exit 1
32+
# ── mode detection ──────────────────────────────────────────────────────────
33+
# Entry-point mode: directly executed AND first arg is an existing .sh file.
34+
# Otherwise act as a sourced setup script.
35+
_RAY_JOB_RUN_SCRIPT=""
36+
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
37+
_RAY_JOB_FIRST_ARG="${1:-}"
38+
if [ -n "$_RAY_JOB_FIRST_ARG" ] && [ -f "$_RAY_JOB_FIRST_ARG" ] && [[ "$_RAY_JOB_FIRST_ARG" == *.sh ]]; then
39+
_RAY_JOB_RUN_SCRIPT="$_RAY_JOB_FIRST_ARG"
40+
shift
41+
else
42+
echo "Usage: $0 <run-script.sh> [extra-args...]" >&2
43+
exit 1
44+
fi
3645
fi
3746

47+
set -eo pipefail
48+
49+
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
50+
3851
# ── clean up residual python/sglang processes (NOT ray) ─────────────────────
3952
# IMPORTANT: Do NOT pkill ray or run ray stop — the cluster is managed externally.
4053
echo "=== Cleaning up residual python/sglang processes ==="
@@ -46,14 +59,13 @@ ray job list | grep RUNNING | grep -v job_id=None | grep -oP "submission_id='\\K
4659
set -x
4760

4861
# ── environment setup ───────────────────────────────────────────────────────
49-
# Use the first GPU node as MASTER_ADDR and SLIME_HOST_IP (prefer head node)
62+
# Use the first GPU node as MASTER_ADDR (prefer head node)
5063
export MASTER_ADDR=$(ray list nodes --format json | jq -r '
5164
map(select(.state == "ALIVE" and (.resources_total.GPU // 0) > 0)) |
5265
sort_by(.is_head_node | not) |
5366
.[0].node_ip
5467
')
5568

56-
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
5769
export PYTHONUNBUFFERED=1
5870
export CUDA_DEVICE_MAX_CONNECTIONS=1
5971
export MEGATRON=${MEGATRON:-/root/Megatron-LM/}
@@ -74,13 +86,12 @@ else
7486
fi
7587
echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)"
7688

77-
# ── delegate to run script ──────────────────────────────────────────────────
78-
echo "=== Launching training script: $RUN_SCRIPT ==="
89+
# ── entrypoint mode & runtime env ──────────────────────────────────────────
7990
export RELAX_ENTRYPOINT_MODE="ray-job"
8091
RAY_DEBUG=${RAY_DEBUG:-"0"}
8192
RAY_DEBUG_POST_MORTEM=${RAY_DEBUG_POST_MORTEM:-"0"}
8293

83-
# Runtime env for ray-job mode (empty, env inherited from Ray cluster)
94+
# Runtime env for ray-job mode (env inherited from Ray cluster)
8495
export RUNTIME_ENV_JSON="{
8596
\"env_vars\": {
8697
\"PYTHONUNBUFFERED\": \"1\",
@@ -93,4 +104,11 @@ export RUNTIME_ENV_JSON="{
93104
\"RAY_DEBUG_POST_MORTEM\": \"${RAY_DEBUG_POST_MORTEM}\"
94105
}
95106
}"
96-
exec bash "$RUN_SCRIPT" "$@"
107+
108+
echo "=== Ray-job environment ready ==="
109+
110+
# ── delegate to run script (entry-point mode only) ─────────────────────────
111+
if [ -n "$_RAY_JOB_RUN_SCRIPT" ]; then
112+
echo "=== Launching training script: $_RAY_JOB_RUN_SCRIPT ==="
113+
exec bash "$_RAY_JOB_RUN_SCRIPT" "$@"
114+
fi

0 commit comments

Comments
 (0)