diff --git a/debug/perf-regression/collector.py b/debug/perf-regression/collector.py index 16b556e..629e885 100644 --- a/debug/perf-regression/collector.py +++ b/debug/perf-regression/collector.py @@ -25,6 +25,8 @@ model_name TEXT NOT NULL, tp_size INTEGER NOT NULL DEFAULT 8, mtp_enabled INTEGER NOT NULL DEFAULT 1, + ep_size INTEGER DEFAULT NULL, + dp_size INTEGER DEFAULT NULL, run_timestamp TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending', error_message TEXT, @@ -190,6 +192,28 @@ def _migrate_drop_unique_constraint(conn: sqlite3.Connection) -> None: logger.info("Migration complete: UNIQUE constraint removed from benchmark_runs") +def _migrate_add_ep_dp_columns(conn: sqlite3.Connection) -> None: + """Add ep_size and dp_size columns if they don't exist (schema migration).""" + cols = { + row[1] for row in conn.execute("PRAGMA table_info(benchmark_runs)").fetchall() + } + if "ep_size" not in cols: + conn.execute("ALTER TABLE benchmark_runs ADD COLUMN ep_size INTEGER DEFAULT NULL") + logger.info("Migrated: added ep_size column to benchmark_runs") + if "dp_size" not in cols: + conn.execute("ALTER TABLE benchmark_runs ADD COLUMN dp_size INTEGER DEFAULT NULL") + logger.info("Migrated: added dp_size column to benchmark_runs") + conn.commit() + + # Recreate index to include ep_size and dp_size + conn.execute("DROP INDEX IF EXISTS idx_runs_image_model") + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_runs_image_model " + "ON benchmark_runs(image_tag, model_name, tp_size, mtp_enabled, ep_size, dp_size)" + ) + conn.commit() + + def init_db(conn: Optional[sqlite3.Connection] = None) -> None: """Create tables if they don't exist.""" close = False @@ -199,6 +223,7 @@ def init_db(conn: Optional[sqlite3.Connection] = None) -> None: conn.executescript(SCHEMA_SQL) _migrate_add_tp_mtp_columns(conn) _migrate_drop_unique_constraint(conn) + _migrate_add_ep_dp_columns(conn) conn.commit() if close: conn.close() @@ -210,12 +235,15 @@ def is_already_benchmarked( image_tag: str, tp_size: int = 8, mtp_enabled: bool = True, + ep_size: Optional[int] = None, + dp_size: Optional[int] = None, ) -> bool: """Check if this image already has a completed benchmark with existing results.""" row = conn.execute( "SELECT status, result_dir FROM benchmark_runs " - "WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ?", - (image_tag, _cfg.MODEL_NAME, tp_size, int(mtp_enabled)), + "WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ? " + "AND ep_size IS ? AND dp_size IS ?", + (image_tag, _cfg.MODEL_NAME, tp_size, int(mtp_enabled), ep_size, dp_size), ).fetchone() if row is None or row["status"] != "completed": return False @@ -237,16 +265,19 @@ def create_run( status: str = "running", tp_size: int = 8, mtp_enabled: bool = True, + ep_size: Optional[int] = None, + dp_size: Optional[int] = None, ) -> int: """Create or reset a benchmark_runs record and return its ID. - If a previous record exists for this image+model+tp+mtp, reset it + If a previous record exists for this image+model+tp+mtp+ep+dp, reset it instead of inserting a duplicate. """ existing = conn.execute( "SELECT id, status FROM benchmark_runs " - "WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ?", - (image_tag, _cfg.MODEL_NAME, tp_size, int(mtp_enabled)), + "WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ? " + "AND ep_size IS ? AND dp_size IS ?", + (image_tag, _cfg.MODEL_NAME, tp_size, int(mtp_enabled), ep_size, dp_size), ).fetchone() now = datetime.now(timezone.utc).isoformat() @@ -273,8 +304,9 @@ def create_run( cur = conn.execute( """INSERT INTO benchmark_runs (image_tag, sglang_version, rocm_version, build_date, - model_name, tp_size, mtp_enabled, run_timestamp, status, result_dir) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + model_name, tp_size, mtp_enabled, ep_size, dp_size, + run_timestamp, status, result_dir) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", ( image_tag, sglang_version, @@ -283,6 +315,8 @@ def create_run( _cfg.MODEL_NAME, tp_size, int(mtp_enabled), + ep_size, + dp_size, now, status, result_dir, diff --git a/debug/perf-regression/config.py b/debug/perf-regression/config.py index 65f2a9b..9a04c53 100755 --- a/debug/perf-regression/config.py +++ b/debug/perf-regression/config.py @@ -3,6 +3,7 @@ import os import re from pathlib import Path +from typing import NamedTuple, Optional # ── Paths ────────────────────────────────────────────────────────────────── BASE_DIR = Path(__file__).resolve().parent @@ -72,18 +73,53 @@ def save_model_config(model_path: str, model_name: str) -> None: MTP_MODE = True # Legacy: used by run_daily.sh; orchestrator uses TP_MTP_VARIANTS CONCURRENCIES = "1,2,4" -# ── TP / MTP variant matrix ────────────────────────────────────────────── +# ── Variant matrix ─────────────────────────────────────────────────────── + + +class VariantConfig(NamedTuple): + tp_size: int + mtp: bool + ep_size: Optional[int] = None + dp_size: Optional[int] = None + + TP_MTP_VARIANTS = [ - (4, False), # TP4 - (4, True), # TP4+MTP - (8, False), # TP8 - (8, True), # TP8+MTP + VariantConfig(4, False), # TP4 + VariantConfig(4, True), # TP4+MTP + VariantConfig(8, False), # TP8 + VariantConfig(8, True), # TP8+MTP ] -def variant_label(tp_size: int, mtp: bool) -> str: +def variant_label(tp_size: int, mtp: bool, ep_size: Optional[int] = None, dp_size: Optional[int] = None) -> str: """Human-readable variant label.""" - return f"TP{tp_size}{'+MTP' if mtp else ''}" + label = f"TP{tp_size}{'+MTP' if mtp else ''}" + if ep_size is not None: + label += f"+EP{ep_size}" + if dp_size is not None: + label += f"+DP{dp_size}" + return label + + +_VARIANT_LABEL_RE = re.compile( + r"^TP(\d+)(\+MTP)?(?:\+EP(\d+))?(?:\+DP(\d+))?$", re.IGNORECASE +) + + +def parse_variant_label(label: str) -> Optional[VariantConfig]: + """Parse a variant label like 'TP4+MTP+EP2+DP4' into a VariantConfig. + + Returns None if the label doesn't match the expected format. + """ + m = _VARIANT_LABEL_RE.match(label) + if not m: + return None + return VariantConfig( + tp_size=int(m.group(1)), + mtp=m.group(2) is not None, + ep_size=int(m.group(3)) if m.group(3) else None, + dp_size=int(m.group(4)) if m.group(4) else None, + ) ACCURACY_MODE = True diff --git a/debug/perf-regression/dashboard.py b/debug/perf-regression/dashboard.py index ee05271..4c553d5 100644 --- a/debug/perf-regression/dashboard.py +++ b/debug/perf-regression/dashboard.py @@ -55,6 +55,8 @@ async def chart_data( model_name: Optional[str] = Query(None), tp_size: Optional[str] = Query(None), mtp_enabled: Optional[str] = Query(None), + ep_size: Optional[str] = Query(None), + dp_size: Optional[str] = Query(None), days: int = Query(30), concurrency: Optional[str] = Query(None), ): @@ -81,6 +83,20 @@ async def chart_data( where_parts.append("br.mtp_enabled = ?") params.append(int(mtp_enabled)) + if ep_size and ep_size != "all": + if ep_size == "none": + where_parts.append("br.ep_size IS NULL") + else: + where_parts.append("br.ep_size = ?") + params.append(int(ep_size)) + + if dp_size and dp_size != "all": + if dp_size == "none": + where_parts.append("br.dp_size IS NULL") + else: + where_parts.append("br.dp_size = ?") + params.append(int(dp_size)) + if days > 0: where_parts.append( f"br.build_date >= strftime('%Y%m%d', 'now', '-{days} days')" @@ -105,6 +121,8 @@ async def chart_data( br.sglang_version, br.tp_size, br.mtp_enabled, + br.ep_size, + br.dp_size, bm.concurrency, bm.output_throughput, bm.total_throughput, @@ -115,7 +133,7 @@ async def chart_data( FROM benchmark_metrics bm JOIN benchmark_runs br ON bm.run_id = br.id WHERE {where_clause}{conc_filter} - ORDER BY br.build_date, br.rocm_version, br.tp_size, br.mtp_enabled, bm.concurrency + ORDER BY br.build_date, br.rocm_version, br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size, bm.concurrency """ rows = conn.execute(metrics_query, params_metrics).fetchall() @@ -128,11 +146,13 @@ async def chart_data( br.sglang_version, br.tp_size, br.mtp_enabled, + br.ep_size, + br.dp_size, ar.accuracy_pct FROM accuracy_results ar JOIN benchmark_runs br ON ar.run_id = br.id WHERE {where_clause} - ORDER BY br.build_date, br.rocm_version, br.tp_size, br.mtp_enabled + ORDER BY br.build_date, br.rocm_version, br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size """ accuracy_rows = conn.execute(accuracy_query, params).fetchall() @@ -203,8 +223,14 @@ def _adjust_shade(hex_color: str, factor: float) -> str: for row in rows: mtp = row["mtp_enabled"] tp = row["tp_size"] + ep = row["ep_size"] + dp = row["dp_size"] mtp_label = "MTP" if mtp else "non-MTP" - key = f"rocm{row['rocm_version']}-tp{tp}-mtp{mtp}-c{row['concurrency']}" + ep_label = f"/EP{ep}" if ep is not None else "" + dp_label = f"/DP{dp}" if dp is not None else "" + ep_key = f"-ep{ep}" if ep is not None else "" + dp_key = f"-dp{dp}" if dp is not None else "" + key = f"rocm{row['rocm_version']}-tp{tp}-mtp{mtp}{ep_key}{dp_key}-c{row['concurrency']}" conc = row["concurrency"] color = rocm_tp_conc_colors.get( (row["rocm_version"], tp, conc), "#607D8B" @@ -220,7 +246,7 @@ def _adjust_shade(hex_color: str, factor: float) -> str: ds = metric_charts[metric_name]["datasets"] if key not in ds: ds[key] = { - "label": f"ROCm {row['rocm_version']} / TP{tp} / {mtp_label} / c={conc}", + "label": f"ROCm {row['rocm_version']} / TP{tp} / {mtp_label}{ep_label}{dp_label} / c={conc}", "data": [], "borderColor": color, "backgroundColor": color + "33", @@ -262,8 +288,14 @@ def _adjust_shade(hex_color: str, factor: float) -> str: for row in accuracy_rows: mtp = row["mtp_enabled"] tp = row["tp_size"] + ep = row["ep_size"] + dp = row["dp_size"] mtp_label = "MTP" if mtp else "non-MTP" - key = f"rocm{row['rocm_version']}-tp{tp}-mtp{mtp}" + ep_label = f"/EP{ep}" if ep is not None else "" + dp_label = f"/DP{dp}" if dp is not None else "" + ep_key = f"-ep{ep}" if ep is not None else "" + dp_key = f"-dp{dp}" if dp is not None else "" + key = f"rocm{row['rocm_version']}-tp{tp}-mtp{mtp}{ep_key}{dp_key}" color = rocm_tp_base.get( (row["rocm_version"], tp), "#607D8B" ) @@ -271,7 +303,7 @@ def _adjust_shade(hex_color: str, factor: float) -> str: if key not in acc_datasets: acc_datasets[key] = { - "label": f"ROCm {row['rocm_version']} / TP{tp} / {mtp_label}", + "label": f"ROCm {row['rocm_version']} / TP{tp} / {mtp_label}{ep_label}{dp_label}", "data": [], "borderColor": color, "backgroundColor": color + "33", @@ -316,6 +348,8 @@ async def get_runs( model_name: Optional[str] = Query(None), tp_size: Optional[str] = Query(None), mtp_enabled: Optional[str] = Query(None), + ep_size: Optional[str] = Query(None), + dp_size: Optional[str] = Query(None), days: int = Query(30), limit: int = Query(100), ): @@ -341,6 +375,20 @@ async def get_runs( where_parts.append("br.mtp_enabled = ?") params.append(int(mtp_enabled)) + if ep_size and ep_size != "all": + if ep_size == "none": + where_parts.append("br.ep_size IS NULL") + else: + where_parts.append("br.ep_size = ?") + params.append(int(ep_size)) + + if dp_size and dp_size != "all": + if dp_size == "none": + where_parts.append("br.dp_size IS NULL") + else: + where_parts.append("br.dp_size = ?") + params.append(int(dp_size)) + if days > 0: where_parts.append( f"br.build_date >= strftime('%Y%m%d', 'now', '-{days} days')" @@ -353,6 +401,7 @@ async def get_runs( SELECT br.id, br.image_tag, br.sglang_version, br.rocm_version, br.build_date, br.model_name, br.tp_size, br.mtp_enabled, + br.ep_size, br.dp_size, br.run_timestamp, br.status, br.error_message, br.duration_total_sec FROM benchmark_runs br @@ -456,6 +505,8 @@ async def get_alerts( model_name: Optional[str] = Query(None), tp_size: Optional[str] = Query(None), mtp_enabled: Optional[str] = Query(None), + ep_size: Optional[str] = Query(None), + dp_size: Optional[str] = Query(None), days: int = Query(30), acknowledged: Optional[bool] = Query(None), ): @@ -481,6 +532,20 @@ async def get_alerts( where_parts.append("br.mtp_enabled = ?") params.append(int(mtp_enabled)) + if ep_size and ep_size != "all": + if ep_size == "none": + where_parts.append("br.ep_size IS NULL") + else: + where_parts.append("br.ep_size = ?") + params.append(int(ep_size)) + + if dp_size and dp_size != "all": + if dp_size == "none": + where_parts.append("br.dp_size IS NULL") + else: + where_parts.append("br.dp_size = ?") + params.append(int(dp_size)) + if days > 0: where_parts.append( f"br.build_date >= strftime('%Y%m%d', 'now', '-{days} days')" @@ -498,7 +563,7 @@ async def get_alerts( ra.current_value, ra.baseline_value, ra.regression_pct, ra.acknowledged, ra.created_at, br.image_tag, br.build_date, br.rocm_version, br.sglang_version, - br.tp_size, br.mtp_enabled + br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size FROM regression_alerts ra JOIN benchmark_runs br ON ra.run_id = br.id WHERE {where_clause} @@ -527,9 +592,9 @@ async def variant_comparison( runs = conn.execute( f""" SELECT br.id, br.model_name, br.image_tag, br.rocm_version, - br.tp_size, br.mtp_enabled + br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size FROM benchmark_runs br WHERE {where} - ORDER BY br.tp_size, br.mtp_enabled + ORDER BY br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size """, params, ).fetchall() @@ -550,6 +615,8 @@ async def variant_comparison( "rocm_version": run["rocm_version"], "tp_size": run["tp_size"], "mtp_enabled": run["mtp_enabled"], + "ep_size": run["ep_size"], + "dp_size": run["dp_size"], "metrics": [dict(m) for m in metrics], "accuracy_pct": accuracy["accuracy_pct"] if accuracy else None, }) diff --git a/debug/perf-regression/orchestrator.py b/debug/perf-regression/orchestrator.py index f954229..66c1535 100644 --- a/debug/perf-regression/orchestrator.py +++ b/debug/perf-regression/orchestrator.py @@ -34,7 +34,9 @@ LOG_DIR, save_model_config, TP_MTP_VARIANTS, + VariantConfig, variant_label, + parse_variant_label, ) from collector import get_connection, init_db, is_already_benchmarked, create_run, update_run_status, ingest_run from discover import discover_images @@ -281,7 +283,8 @@ def _watcher(): return t -def run_benchmark(image: str, result_dir: Path, tp_size: int = 8, mtp: bool = True) -> tuple[str, str]: +def run_benchmark(image: str, result_dir: Path, tp_size: int = 8, mtp: bool = True, + ep_size: int = None, dp_size: int = None) -> tuple[str, str]: """Run the benchmark script and return (stdout, container_name). Passes all flags to suppress interactive prompts. @@ -310,6 +313,14 @@ def run_benchmark(image: str, result_dir: Path, tp_size: int = 8, mtp: bool = Tr else: cmd.append("--no-mtp") + extra_server_args = [] + if ep_size is not None: + extra_server_args.extend(["--ep-size", str(ep_size)]) + if dp_size is not None: + extra_server_args.extend(["--dp-size", str(dp_size)]) + if extra_server_args: + cmd.extend(["--server-extra-args", " ".join(extra_server_args)]) + if _cfg.ACCURACY_MODE: if _cfg.ACCURACY_ONLY: cmd.append("--accuracy-only") @@ -551,10 +562,12 @@ def process_image(conn, image_info, dry_run: bool = False, variants=None) -> boo if not pull_image(image): return False - for tp_size, mtp in active_variants: - label = variant_label(tp_size, mtp) + for v in active_variants: + tp_size, mtp, ep_size, dp_size = v.tp_size, v.mtp, v.ep_size, v.dp_size + label = variant_label(tp_size, mtp, ep_size=ep_size, dp_size=dp_size) - if is_already_benchmarked(conn, tag, tp_size=tp_size, mtp_enabled=mtp): + if is_already_benchmarked(conn, tag, tp_size=tp_size, mtp_enabled=mtp, + ep_size=ep_size, dp_size=dp_size): logger.info("Skipping %s [%s] (already benchmarked)", tag, label) continue @@ -584,13 +597,16 @@ def process_image(conn, image_info, dry_run: bool = False, variants=None) -> boo status="running", tp_size=tp_size, mtp_enabled=mtp, + ep_size=ep_size, + dp_size=dp_size, ) _active_run_id = run_id # Run benchmark start_time = time.monotonic() stdout, container_name = run_benchmark( - image, result_dir, tp_size=tp_size, mtp=mtp + image, result_dir, tp_size=tp_size, mtp=mtp, + ep_size=ep_size, dp_size=dp_size, ) duration = time.monotonic() - start_time @@ -705,7 +721,19 @@ def main(): "--variants", nargs="+", default=None, - help="Only run specific variants (e.g., TP8 TP8+MTP). Default: all", + help="Only run specific variants (e.g., TP8 TP8+MTP TP8+EP2+DP4). Default: all", + ) + parser.add_argument( + "--ep-size", + type=int, + default=None, + help="Add EP (expert parallelism) size to all variants", + ) + parser.add_argument( + "--dp-size", + type=int, + default=None, + help="Add DP (data parallelism) size to all variants", ) args = parser.parse_args() @@ -740,15 +768,28 @@ def _strip_ansi(s: str) -> str: lookback_max, lookback_min = _parse_lookback_days(args.lookback_days) # Parse --variants filter + # Variants can be specified as labels like TP4, TP8+MTP, TP4+DP2, TP8+EP2+DP4. + # Labels are parsed directly — no need for separate --ep-size/--dp-size flags. + # If --ep-size/--dp-size are given without --variants, they apply to the + # default variant matrix as overrides. active_variants = None if args.variants: + active_variants = [] + for label in args.variants: + parsed = parse_variant_label(label) + if parsed is None: + parser.error( + f"Invalid variant label: '{label}'. " + f"Expected format: TP[+MTP][+EP][+DP]" + ) + active_variants.append(parsed) + elif args.ep_size is not None or args.dp_size is not None: active_variants = [ - (tp, mtp) for tp, mtp in TP_MTP_VARIANTS - if variant_label(tp, mtp) in args.variants + VariantConfig(v.tp_size, v.mtp, + ep_size=args.ep_size if args.ep_size is not None else v.ep_size, + dp_size=args.dp_size if args.dp_size is not None else v.dp_size) + for v in TP_MTP_VARIANTS ] - if not active_variants: - available = [variant_label(t, m) for t, m in TP_MTP_VARIANTS] - parser.error(f"No matching variants. Available: {available}") # Validate rocm versions for v in args.rocm_versions: @@ -827,12 +868,14 @@ def _strip_ansi(s: str) -> str: ) rerun_variants = active_variants or TP_MTP_VARIANTS for tag in args.force_rerun: - for tp, mtp in rerun_variants: - logger.info("Force re-running: %s [%s]", tag, variant_label(tp, mtp)) + for v in rerun_variants: + label = variant_label(v.tp_size, v.mtp, ep_size=v.ep_size, dp_size=v.dp_size) + logger.info("Force re-running: %s [%s]", tag, label) conn.execute( "DELETE FROM benchmark_runs " - "WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ?", - (tag, _cfg.MODEL_NAME, tp, int(mtp)), + "WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ? " + "AND ep_size IS ? AND dp_size IS ?", + (tag, _cfg.MODEL_NAME, v.tp_size, int(v.mtp), v.ep_size, v.dp_size), ) conn.commit() diff --git a/debug/perf-regression/regression.py b/debug/perf-regression/regression.py index dc226bc..c051936 100644 --- a/debug/perf-regression/regression.py +++ b/debug/perf-regression/regression.py @@ -27,7 +27,8 @@ def detect_regressions( """ # Get current run info run = conn.execute( - "SELECT id, rocm_version, build_date, model_name, tp_size, mtp_enabled " + "SELECT id, rocm_version, build_date, model_name, tp_size, mtp_enabled, " + "ep_size, dp_size " "FROM benchmark_runs WHERE id = ?", (run_id,), ).fetchone() @@ -39,6 +40,8 @@ def detect_regressions( model_name = run["model_name"] tp_size = run["tp_size"] mtp_enabled = run["mtp_enabled"] + ep_size = run["ep_size"] + dp_size = run["dp_size"] alerts = [] # Get concurrencies for this run @@ -57,13 +60,15 @@ def detect_regressions( alert = _check_metric_regression( conn, run_id, rocm_version, model_name, tp_size, mtp_enabled, metric_name, conc, direction, threshold, + ep_size=ep_size, dp_size=dp_size, ) if alert: alerts.append(alert) # Check accuracy regression accuracy_alert = _check_accuracy_regression( - conn, run_id, rocm_version, model_name, tp_size, mtp_enabled + conn, run_id, rocm_version, model_name, tp_size, mtp_enabled, + ep_size=ep_size, dp_size=dp_size, ) if accuracy_alert: alerts.append(accuracy_alert) @@ -104,6 +109,8 @@ def _check_metric_regression( concurrency: int, direction: str, threshold_pct: float, + ep_size: Optional[int] = None, + dp_size: Optional[int] = None, ) -> Optional[dict]: """Check if a metric at a specific concurrency has regressed.""" # Get current value @@ -128,6 +135,8 @@ def _check_metric_regression( AND br.model_name = ? AND br.tp_size = ? AND br.mtp_enabled = ? + AND br.ep_size IS ? + AND br.dp_size IS ? AND br.status = 'completed' AND br.build_date < ? AND bm.concurrency = ? @@ -135,7 +144,7 @@ def _check_metric_regression( ORDER BY br.build_date DESC LIMIT ?""", (rocm_version, model_name, tp_size, mtp_enabled, - build_date, concurrency, REGRESSION_WINDOW), + ep_size, dp_size, build_date, concurrency, REGRESSION_WINDOW), ).fetchall() if len(previous_values) < 2: @@ -181,6 +190,8 @@ def _check_accuracy_regression( model_name: str, tp_size: int, mtp_enabled: int, + ep_size: Optional[int] = None, + dp_size: Optional[int] = None, ) -> Optional[dict]: """Check if accuracy has regressed beyond the threshold.""" current_row = conn.execute( @@ -203,13 +214,15 @@ def _check_accuracy_regression( AND br.model_name = ? AND br.tp_size = ? AND br.mtp_enabled = ? + AND br.ep_size IS ? + AND br.dp_size IS ? AND br.status = 'completed' AND br.build_date < ? AND ar.accuracy_pct IS NOT NULL ORDER BY br.build_date DESC LIMIT ?""", (rocm_version, model_name, tp_size, mtp_enabled, - build_date, REGRESSION_WINDOW), + ep_size, dp_size, build_date, REGRESSION_WINDOW), ).fetchall() if len(previous_values) < 2: diff --git a/debug/perf-regression/templates/index.html b/debug/perf-regression/templates/index.html index b192513..eb5fd77 100644 --- a/debug/perf-regression/templates/index.html +++ b/debug/perf-regression/templates/index.html @@ -66,6 +66,26 @@

SGLang ROCm Performance Regression Dashboard

+
+ + +
+
+ + +
@@ -130,6 +150,8 @@

Run History

Model TP MTP + EP + DP SGLang ROCm Status @@ -141,7 +163,7 @@

Run History

- Loading... + Loading... @@ -154,6 +176,8 @@

Regression Alerts

ROCm TP MTP + EP + DP Metric Concurrency Current @@ -163,7 +187,7 @@

Regression Alerts

- Loading... + Loading... @@ -184,6 +208,8 @@

Regression Alerts

concurrency: document.getElementById('filter-concurrency').value, tp_size: document.getElementById('filter-tp').value, mtp_enabled: document.getElementById('filter-mtp').value, + ep_size: document.getElementById('filter-ep').value, + dp_size: document.getElementById('filter-dp').value, }; } @@ -196,6 +222,8 @@

Regression Alerts

concurrency: f.concurrency, tp_size: f.tp_size, mtp_enabled: f.mtp_enabled, + ep_size: f.ep_size, + dp_size: f.dp_size, }); const resp = await fetch(`/api/chart-data?${params}`); return await resp.json(); @@ -209,6 +237,8 @@

Regression Alerts

model_name: f.model_name, tp_size: f.tp_size, mtp_enabled: f.mtp_enabled, + ep_size: f.ep_size, + dp_size: f.dp_size, }); const resp = await fetch(`/api/alerts?${params}`); return await resp.json(); @@ -222,6 +252,8 @@

Regression Alerts

model_name: f.model_name, tp_size: f.tp_size, mtp_enabled: f.mtp_enabled, + ep_size: f.ep_size, + dp_size: f.dp_size, }); const resp = await fetch(`/api/runs?${params}`); return await resp.json(); @@ -307,7 +339,7 @@

Regression Alerts

function renderAlerts(alerts) { const tbody = document.getElementById('alerts-body'); if (!alerts || alerts.length === 0) { - tbody.innerHTML = 'No regression alerts'; + tbody.innerHTML = 'No regression alerts'; return; } tbody.innerHTML = alerts.map(a => { @@ -319,6 +351,8 @@

Regression Alerts

${a.rocm_version} ${a.tp_size != null ? a.tp_size : '-'} ${a.mtp_enabled ? 'Yes' : 'No'} + ${a.ep_size != null ? a.ep_size : '-'} + ${a.dp_size != null ? a.dp_size : '-'} ${a.metric_name} ${a.concurrency !== null ? a.concurrency : 'N/A'} ${a.current_value !== null ? a.current_value.toFixed(2) : 'N/A'} @@ -332,7 +366,7 @@

Regression Alerts

function renderRuns(runs) { const tbody = document.getElementById('runs-body'); if (!runs || runs.length === 0) { - tbody.innerHTML = 'No benchmark runs'; + tbody.innerHTML = 'No benchmark runs'; return; } tbody.innerHTML = runs.map(r => { @@ -346,7 +380,7 @@

Regression Alerts

let metricsHtml = ''; if (r.metrics && r.metrics.length > 0) { metricsHtml = ` - + @@ -379,7 +413,7 @@

Regression Alerts

if (r.versions && r.versions.length > 0) { const versionsId = `versions-${r.id}`; versionsHtml = ` -