From 670155000cac79f2b53e19c33e1006c96ca78973 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Tue, 24 Feb 2026 03:45:27 -0600 Subject: [PATCH 1/3] [Feature] Support accuracy only mode --- benchmark/run-local-benchmark-e2e.sh | 77 +++++++++++++++------------ debug/perf-regression/config.py | 3 +- debug/perf-regression/orchestrator.py | 19 +++++-- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/benchmark/run-local-benchmark-e2e.sh b/benchmark/run-local-benchmark-e2e.sh index ca716c4..f650c6a 100755 --- a/benchmark/run-local-benchmark-e2e.sh +++ b/benchmark/run-local-benchmark-e2e.sh @@ -29,6 +29,7 @@ Common options: --keep-container Do not stop/remove container on exit. --no-sudo-docker Use docker directly instead of sudo docker. --accuracy Run GSM8K accuracy benchmark after perf bench. + --accuracy-only Run GSM8K accuracy benchmark only (skip perf bench). --no-accuracy Skip accuracy benchmark (skip interactive prompt). --accuracy-num-questions Number of GSM8K questions. Default: 2000 --accuracy-parallel GSM8K parallel requests. Default: 1000 @@ -106,6 +107,7 @@ USE_SUDO_DOCKER=1 HOST_HOME_DIR="" MTP_MODE="" ACCURACY_MODE="" +ACCURACY_ONLY=0 ACCURACY_NUM_QUESTIONS=2000 ACCURACY_PARALLEL=1000 ACCURACY_NUM_SHOTS=5 @@ -189,6 +191,11 @@ while [[ $# -gt 0 ]]; do ACCURACY_MODE=1 shift ;; + --accuracy-only) + ACCURACY_MODE=1 + ACCURACY_ONLY=1 + shift + ;; --no-accuracy) ACCURACY_MODE=0 shift @@ -539,42 +546,46 @@ if (( ACCURACY_MODE == 1 )); then | tee -a "$HOST_ACCURACY_LOG" fi -for c in "${CONCURRENCY_ARRAY[@]}"; do - NUM_PROMPTS=$((c * PROMPTS_MULTIPLIER)) - CONTAINER_BENCH_LOG="${CONTAINER_RESULT_DIR}/bench_c${c}.log" - - BENCH_ARGS=( - python3 -m sglang.bench_serving - --backend sglang - --host localhost - --port "$SERVER_PORT" - --model "$MODEL_PATH" - --dataset-name "$DATASET_NAME" - --random-input-len "$RANDOM_INPUT_LEN" - --random-output-len "$RANDOM_OUTPUT_LEN" - --random-range-ratio "$RANDOM_RANGE_RATIO" - --max-concurrency "$c" - --num-prompts "$NUM_PROMPTS" - --disable-tqdm - --output-file "$CONTAINER_BENCH_JSONL" - ) +if (( ACCURACY_ONLY == 1 )); then + log "Accuracy-only mode: skipping perf benchmark" +else + for c in "${CONCURRENCY_ARRAY[@]}"; do + NUM_PROMPTS=$((c * PROMPTS_MULTIPLIER)) + CONTAINER_BENCH_LOG="${CONTAINER_RESULT_DIR}/bench_c${c}.log" + + BENCH_ARGS=( + python3 -m sglang.bench_serving + --backend sglang + --host localhost + --port "$SERVER_PORT" + --model "$MODEL_PATH" + --dataset-name "$DATASET_NAME" + --random-input-len "$RANDOM_INPUT_LEN" + --random-output-len "$RANDOM_OUTPUT_LEN" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --max-concurrency "$c" + --num-prompts "$NUM_PROMPTS" + --disable-tqdm + --output-file "$CONTAINER_BENCH_JSONL" + ) - if (( PROFILE_MODE == 1 )); then - BENCH_ARGS+=(--profile) - fi + if (( PROFILE_MODE == 1 )); then + BENCH_ARGS+=(--profile) + fi - if [[ -n "$BENCH_EXTRA_ARGS" ]]; then - # shellcheck disable=SC2206 - BENCH_EXTRA_ARRAY=($BENCH_EXTRA_ARGS) - BENCH_ARGS+=("${BENCH_EXTRA_ARRAY[@]}") - fi + if [[ -n "$BENCH_EXTRA_ARGS" ]]; then + # shellcheck disable=SC2206 + BENCH_EXTRA_ARRAY=($BENCH_EXTRA_ARGS) + BENCH_ARGS+=("${BENCH_EXTRA_ARRAY[@]}") + fi - BENCH_CMD="$(quote_join "${BENCH_ARGS[@]}")" - log "Running benchmark: concurrency=${c}, num_prompts=${NUM_PROMPTS}" - docker_cmd exec "$CONTAINER_NAME" bash -lc \ - "cd /sgl-workspace/sglang/python && ${BENCH_CMD} 2>&1 | tee -a $(quote_one "$CONTAINER_BENCH_LOG")" \ - | tee -a "$HOST_CLIENT_LOG" -done + BENCH_CMD="$(quote_join "${BENCH_ARGS[@]}")" + log "Running benchmark: concurrency=${c}, num_prompts=${NUM_PROMPTS}" + docker_cmd exec "$CONTAINER_NAME" bash -lc \ + "cd /sgl-workspace/sglang/python && ${BENCH_CMD} 2>&1 | tee -a $(quote_one "$CONTAINER_BENCH_LOG")" \ + | tee -a "$HOST_CLIENT_LOG" + done +fi if (( PROFILE_MODE == 1 )); then log "Collecting TP0 trace from /tmp" diff --git a/debug/perf-regression/config.py b/debug/perf-regression/config.py index ad32be8..65f2a9b 100755 --- a/debug/perf-regression/config.py +++ b/debug/perf-regression/config.py @@ -74,8 +74,6 @@ def save_model_config(model_path: str, model_name: str) -> None: # ── TP / MTP variant matrix ────────────────────────────────────────────── TP_MTP_VARIANTS = [ - (2, False), # TP2 - (2, True), # TP2+MTP (4, False), # TP4 (4, True), # TP4+MTP (8, False), # TP8 @@ -89,6 +87,7 @@ def variant_label(tp_size: int, mtp: bool) -> str: ACCURACY_MODE = True +ACCURACY_ONLY = False ACCURACY_NUM_QUESTIONS = 2000 ACCURACY_PARALLEL = 1000 ACCURACY_NUM_SHOTS = 1 diff --git a/debug/perf-regression/orchestrator.py b/debug/perf-regression/orchestrator.py index 0861ea6..f954229 100644 --- a/debug/perf-regression/orchestrator.py +++ b/debug/perf-regression/orchestrator.py @@ -19,7 +19,6 @@ CONCURRENCIES, USE_SUDO_DOCKER, MTP_MODE, - ACCURACY_MODE, ACCURACY_NUM_QUESTIONS, ACCURACY_PARALLEL, ACCURACY_NUM_SHOTS, @@ -311,9 +310,12 @@ def run_benchmark(image: str, result_dir: Path, tp_size: int = 8, mtp: bool = Tr else: cmd.append("--no-mtp") - if ACCURACY_MODE: + if _cfg.ACCURACY_MODE: + if _cfg.ACCURACY_ONLY: + cmd.append("--accuracy-only") + else: + cmd.append("--accuracy") cmd.extend([ - "--accuracy", "--accuracy-num-questions", str(ACCURACY_NUM_QUESTIONS), "--accuracy-parallel", str(ACCURACY_PARALLEL), "--accuracy-num-shots", str(ACCURACY_NUM_SHOTS), @@ -693,6 +695,12 @@ def main(): help="Force re-run image tags (deletes existing records). " "If no tags specified, re-runs all images found in --lookback-days.", ) + parser.add_argument( + "--accuracy-only", + action="store_true", + default=False, + help="Run accuracy benchmark only, skip perf benchmark", + ) parser.add_argument( "--variants", nargs="+", @@ -723,6 +731,11 @@ def _strip_ansi(s: str) -> str: # Persist for next run save_model_config(_cfg.MODEL_PATH, _cfg.MODEL_NAME) + # Apply --accuracy-only override + if args.accuracy_only: + _cfg.ACCURACY_MODE = True + _cfg.ACCURACY_ONLY = True + # Parse --lookback-days (single value or range) lookback_max, lookback_min = _parse_lookback_days(args.lookback_days) From ed9d79a39300e0e3421a7218fa837897f7889f73 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Tue, 24 Feb 2026 03:49:49 -0600 Subject: [PATCH 2/3] [Fix] Change key constraint --- debug/perf-regression/collector.py | 69 +++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/debug/perf-regression/collector.py b/debug/perf-regression/collector.py index 4c6dd5b..16b556e 100644 --- a/debug/perf-regression/collector.py +++ b/debug/perf-regression/collector.py @@ -29,8 +29,7 @@ status TEXT NOT NULL DEFAULT 'pending', error_message TEXT, result_dir TEXT, - duration_total_sec REAL, - UNIQUE(image_tag, model_name, tp_size, mtp_enabled) + duration_total_sec REAL ); CREATE TABLE IF NOT EXISTS benchmark_metrics ( @@ -126,6 +125,71 @@ def _migrate_add_tp_mtp_columns(conn: sqlite3.Connection) -> None: conn.commit() +def _migrate_drop_unique_constraint(conn: sqlite3.Connection) -> None: + """Remove UNIQUE(image_tag, model_name, tp_size, mtp_enabled) from benchmark_runs. + + This allows multiple runs for the same image+model+tp+mtp combination, + enabling future schema extensions without unique-key conflicts. + Deduplication is handled in application code (is_already_benchmarked). + + Idempotent: skips if the table already lacks the UNIQUE constraint. + Uses SQLite rename-and-copy since ALTER TABLE cannot drop constraints. + """ + table_sql = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='benchmark_runs'" + ).fetchone() + if table_sql is None: + return + if "UNIQUE" not in table_sql[0]: + return + + logger.info("Migrating benchmark_runs: dropping UNIQUE constraint...") + + conn.execute("ALTER TABLE benchmark_runs RENAME TO benchmark_runs_old") + conn.executescript(""" + CREATE TABLE benchmark_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + image_tag TEXT NOT NULL, + sglang_version TEXT, + rocm_version TEXT, + build_date TEXT, + model_name TEXT NOT NULL, + tp_size INTEGER NOT NULL DEFAULT 8, + mtp_enabled INTEGER NOT NULL DEFAULT 1, + run_timestamp TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + error_message TEXT, + result_dir TEXT, + duration_total_sec REAL + ); + """) + conn.execute(""" + INSERT INTO benchmark_runs + (id, image_tag, sglang_version, rocm_version, build_date, + model_name, tp_size, mtp_enabled, + run_timestamp, status, error_message, result_dir, duration_total_sec) + SELECT + id, image_tag, sglang_version, rocm_version, build_date, + model_name, tp_size, mtp_enabled, + run_timestamp, status, error_message, result_dir, duration_total_sec + FROM benchmark_runs_old + """) + conn.execute("DROP TABLE benchmark_runs_old") + + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_runs_rocm_status ON benchmark_runs(rocm_version, status)" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_runs_build_date ON benchmark_runs(build_date)" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_runs_image_model ON benchmark_runs(image_tag, model_name, tp_size, mtp_enabled)" + ) + + conn.commit() + logger.info("Migration complete: UNIQUE constraint removed from benchmark_runs") + + def init_db(conn: Optional[sqlite3.Connection] = None) -> None: """Create tables if they don't exist.""" close = False @@ -134,6 +198,7 @@ def init_db(conn: Optional[sqlite3.Connection] = None) -> None: close = True conn.executescript(SCHEMA_SQL) _migrate_add_tp_mtp_columns(conn) + _migrate_drop_unique_constraint(conn) conn.commit() if close: conn.close() From 0fafb1fee2ff93bea6ab559a5013596e3f42222f Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Tue, 24 Feb 2026 15:03:05 +0000 Subject: [PATCH 3/3] [Chore] Add /commit and /pr slash command skills and document git workflow in CLAUDE.md --- .claude/commands/commit.md | 87 +++++++++++++++++++++++++++++++++ .claude/commands/pr.md | 99 ++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 9 +++- 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 .claude/commands/commit.md create mode 100644 .claude/commands/pr.md diff --git a/.claude/commands/commit.md b/.claude/commands/commit.md new file mode 100644 index 0000000..366101d --- /dev/null +++ b/.claude/commands/commit.md @@ -0,0 +1,87 @@ +--- +description: Stage, branch, and commit changes following repo conventions (auto-detects commit-msg hook tag format) +--- + +# Commit Changes + +Follow these steps precisely to commit the current changes. + +## Step 1: Check current branch, status, and commit-msg hook + +Run these in parallel: +- `git status` to see all changed/untracked files +- `git branch --show-current` to get the current branch name +- `git diff` to see unstaged changes +- `git diff --cached` to see staged changes +- `git log --oneline -5` to see recent commit style +- Read the active commit-msg hook to discover allowed tag formats: + 1. Run `git config --get core.hooksPath` to find the hooks directory (defaults to `.git/hooks` if unset) + 2. Read the `commit-msg` file from that directory (e.g., `cat $(git rev-parse --show-toplevel)/.githooks/commit-msg` or `.git/hooks/commit-msg`) + 3. Parse the hook to extract the allowed tags (look for patterns like `Feature|Fix|Refactor|...`) + +## Step 2: Determine the commit message tag format + +Based on the commit-msg hook analysis from Step 1: + +- **If a commit-msg hook exists and enforces specific tags** (e.g., `[Feature]`, `[Fix]`, `[Refactor]`, `[Docs]`, `[Test]`, `[CI]`, `[Chore]`, `[Perf]`): + Use the repo's required format. Pick the tag that best matches the nature of the change: + - `[Feature]` — new functionality + - `[Fix]` — bug fix + - `[Refactor]` — code restructuring without behavior change + - `[Docs]` — documentation only + - `[Test]` — adding or updating tests + - `[CI]` — CI/CD pipeline changes + - `[Chore]` — maintenance, dependencies, tooling + - `[Perf]` — performance improvement + +- **If no commit-msg hook exists or it doesn't enforce a tag format**: + Use `[AMD]` as the default prefix. + +## Step 3: Ensure we are on a feature branch (NOT main/master) + +If the current branch is `main` or `master`: +1. Determine a short, descriptive branch name based on the changes (e.g., `fix-mla-bf16-attention`, `add-rocm-triton-kernel`) +2. Create and switch to the new branch: `git checkout -b ` + +If already on a feature branch, stay on it. + +## Step 4: Stage the changes + +- Stage only the relevant changed files by name (e.g., `git add file1.py file2.py`) +- Do NOT use `git add -A` or `git add .` unless the user explicitly asks +- Do NOT stage files that look like they contain secrets (.env, credentials, tokens, etc.) +- If unsure which files to stage, ask the user + +## Step 5: Write the commit message + +The commit message MUST follow this format: +- Start with the tag determined in Step 2 (e.g., `[Fix]` or `[AMD]`) +- Followed by a single concise sentence describing what was changed and why +- Example (with repo hook): `[Fix] Resolve bf16 type casting in MLA decode attention for dp-attention mode` +- Example (without hook): `[AMD] Fix bf16 type casting in MLA decode attention for dp-attention mode` +- Do NOT include `Co-Authored-By` or any other trailers — they are forbidden by project convention + +If the user provided `$ARGUMENTS`, incorporate that into the commit message description. + +If no arguments were provided, analyze the diff to write an appropriate one-sentence summary. + +## Step 6: Create the commit + +Always use `--author` to set the commit author explicitly: + +```bash +git commit --author="jacky.cheng " -m "[Tag] " +``` + +## Step 7: Verify + +Run `git status` and `git log --oneline -3` to confirm the commit was created successfully. Report the branch name and commit hash to the user. + +## Important Rules + +- NEVER include Co-Authored-By or any trailers in the commit message +- NEVER amend a previous commit unless the user explicitly asks +- NEVER force push +- NEVER skip pre-commit hooks +- If a pre-commit hook fails, fix the issue, re-stage, and create a NEW commit +- NEVER commit .env files, credentials, or secrets diff --git a/.claude/commands/pr.md b/.claude/commands/pr.md new file mode 100644 index 0000000..c9bcff2 --- /dev/null +++ b/.claude/commands/pr.md @@ -0,0 +1,99 @@ +--- +description: Create a GitHub Pull Request following SGLang project conventions with proper sections +--- + +# Create a Pull Request for SGLang + +Follow these steps precisely to create a well-formatted PR. + +## Step 1: Gather context + +Run these in parallel: +- `git status` to check for uncommitted changes +- `git branch --show-current` to get current branch +- `git log --oneline main..HEAD` to see all commits on this branch +- `git diff main...HEAD` to see all changes vs main +- Check if the branch has a remote tracking branch: `git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null` + +If there are uncommitted changes, warn the user and ask if they want to commit first (suggest using `/commit`). + +## Step 2: Push the branch + +If the branch is not pushed or is behind the remote: +- Push with: `git push -u origin ` + +## Step 3: Analyze the changes + +Read through all the diffs and commits to understand: +- What was the motivation / bug being fixed / feature being added? +- What specific files and code were modified? +- Does this change affect model accuracy (kernel changes, model forward code)? +- Does this change affect inference speed / performance? + +## Step 4: Determine the category of change + +Based on the analysis, determine if this PR: +- **Affects accuracy**: Changes to kernels, model forward code, attention mechanisms, etc. -> Need accuracy test results +- **Affects performance**: Changes to scheduling, batching, memory management, kernels, etc. -> Need benchmark results +- **Neither**: Documentation, CI, refactoring with no behavioral change, etc. + +## Step 5: Create the PR + +Use `gh pr create` with the following template. Fill in each section based on your analysis. + +If the user provided `$ARGUMENTS`, use that as the PR title. Otherwise, craft a concise title (under 70 chars) starting with `[AMD]`. + +```bash +gh pr create --title "[AMD] " --body "$(cat <<'EOF' +## Motivation + + + +## Modifications + + + +## Accuracy Tests + + + + +## Benchmarking and Profiling + + + + +## Checklist + +- [ ] Format your code according to the [Format code with pre-commit](https://docs.sglang.io/developer_guide/contribution_guide.html#format-code-with-pre-commit). +- [ ] Add unit tests according to the [Run and add unit tests](https://docs.sglang.io/developer_guide/contribution_guide.html#run-and-add-unit-tests). +- [ ] Update documentation according to [Write documentations](https://docs.sglang.io/developer_guide/contribution_guide.html#write-documentations). +- [ ] Provide accuracy and speed benchmark results according to [Test the accuracy](https://docs.sglang.io/developer_guide/contribution_guide.html#test-the-accuracy) and [Benchmark the speed](https://docs.sglang.io/developer_guide/contribution_guide.html#benchmark-the-speed). +- [ ] Follow the SGLang code style [guidance](https://docs.sglang.io/developer_guide/contribution_guide.html#code-style-guidance). + +## Review Process + +1. Ping Merge Oncalls to start the PR flow. See the [PR Merge Process](https://github.com/sgl-project/sglang/blob/main/.github/MAINTAINER.md#pull-request-merge-process). +2. Get approvals from [CODEOWNERS](https://github.com/sgl-project/sglang/blob/main/.github/CODEOWNERS) and other reviewers. +3. Trigger CI tests with [comments](https://docs.sglang.io/developer_guide/contribution_guide.html#how-to-trigger-ci-tests) or contact authorized users to do so. + - `/tag-run-ci-label`, `/rerun-failed-ci`, `/tag-and-rerun-ci` +4. After green CI and required approvals, ask Merge Oncalls to merge. +EOF +)" +``` + +## Step 6: Report back + +After the PR is created, show the user: +- The PR URL +- A summary of what was included +- Remind them about the review process (ping merge oncalls, get CODEOWNER approvals, trigger CI) +- If accuracy-related: remind them to run accuracy tests and post results +- If performance-related: remind them to run benchmarks and post results + +## Important Rules + +- NEVER force push +- NEVER create PRs to branches other than `main` unless the user explicitly asks +- Always use the full template with all sections filled in +- The PR title should start with `[AMD]` and be under 70 characters diff --git a/CLAUDE.md b/CLAUDE.md index ba4ce64..2ad7f1e 100755 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,4 +93,11 @@ env.sh — Central environment config (HOST_HOME, AGENT_BOX_DIR) ## Trace Analyzer -When debugging or adding features to `trace_analyzer.py`, use the `/trace-analyzer` skill command for detailed guidance on pattern tables, half-layer handling, and diagnostic steps. See `profile/trace-analyzer.md`. \ No newline at end of file +When debugging or adding features to `trace_analyzer.py`, use the `/trace-analyzer` skill command for detailed guidance on pattern tables, half-layer handling, and diagnostic steps. See `profile/trace-analyzer.md`. + +## Git Workflow Skills + +Two slash commands are available for streamlined git workflows: + +- **`/commit`** — Stage changes, ensure you're on a feature branch (creates one if on main), and commit with an `[AMD]` prefixed message. Usage: `/commit` or `/commit `. +- **`/pr`** — Push the branch and create a GitHub PR with the full SGLang template (Motivation, Modifications, Accuracy Tests, Benchmarking, Checklist, Review Process). Usage: `/pr` or `/pr `. \ No newline at end of file