Skip to content

[Feature] Add ep_size and dp_size as new variant dimensions with DB #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yichiche merged 1 commit into from
Feb 24, 2026
Merged

[Feature] Add ep_size and dp_size as new variant dimensions with DB #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 41 additions & 7 deletions debug/perf-regression/collector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
model_name TEXT NOT NULL,
tp_size INTEGER NOT NULL DEFAULT 8,
mtp_enabled INTEGER NOT NULL DEFAULT 1,
ep_size INTEGER DEFAULT NULL,
dp_size INTEGER DEFAULT NULL,
run_timestamp TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
error_message TEXT,
Expand Down Expand Up @@ -190,6 +192,28 @@ def _migrate_drop_unique_constraint(conn: sqlite3.Connection) -> None:
logger.info("Migration complete: UNIQUE constraint removed from benchmark_runs")


def _migrate_add_ep_dp_columns(conn: sqlite3.Connection) -> None:
"""Add ep_size and dp_size columns if they don't exist (schema migration)."""
cols = {
row[1] for row in conn.execute("PRAGMA table_info(benchmark_runs)").fetchall()
}
if "ep_size" not in cols:
conn.execute("ALTER TABLE benchmark_runs ADD COLUMN ep_size INTEGER DEFAULT NULL")
logger.info("Migrated: added ep_size column to benchmark_runs")
if "dp_size" not in cols:
conn.execute("ALTER TABLE benchmark_runs ADD COLUMN dp_size INTEGER DEFAULT NULL")
logger.info("Migrated: added dp_size column to benchmark_runs")
conn.commit()

# Recreate index to include ep_size and dp_size
conn.execute("DROP INDEX IF EXISTS idx_runs_image_model")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_runs_image_model "
"ON benchmark_runs(image_tag, model_name, tp_size, mtp_enabled, ep_size, dp_size)"
)
conn.commit()


def init_db(conn: Optional[sqlite3.Connection] = None) -> None:
"""Create tables if they don't exist."""
close = False
Expand All @@ -199,6 +223,7 @@ def init_db(conn: Optional[sqlite3.Connection] = None) -> None:
conn.executescript(SCHEMA_SQL)
_migrate_add_tp_mtp_columns(conn)
_migrate_drop_unique_constraint(conn)
_migrate_add_ep_dp_columns(conn)
conn.commit()
if close:
conn.close()
Expand All @@ -210,12 +235,15 @@ def is_already_benchmarked(
image_tag: str,
tp_size: int = 8,
mtp_enabled: bool = True,
ep_size: Optional[int] = None,
dp_size: Optional[int] = None,
) -> bool:
"""Check if this image already has a completed benchmark with existing results."""
row = conn.execute(
"SELECT status, result_dir FROM benchmark_runs "
"WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ?",
(image_tag, _cfg.MODEL_NAME, tp_size, int(mtp_enabled)),
"WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ? "
"AND ep_size IS ? AND dp_size IS ?",
(image_tag, _cfg.MODEL_NAME, tp_size, int(mtp_enabled), ep_size, dp_size),
).fetchone()
if row is None or row["status"] != "completed":
return False
Expand All @@ -237,16 +265,19 @@ def create_run(
status: str = "running",
tp_size: int = 8,
mtp_enabled: bool = True,
ep_size: Optional[int] = None,
dp_size: Optional[int] = None,
) -> int:
"""Create or reset a benchmark_runs record and return its ID.

If a previous record exists for this image+model+tp+mtp, reset it
If a previous record exists for this image+model+tp+mtp+ep+dp, reset it
instead of inserting a duplicate.
"""
existing = conn.execute(
"SELECT id, status FROM benchmark_runs "
"WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ?",
(image_tag, _cfg.MODEL_NAME, tp_size, int(mtp_enabled)),
"WHERE image_tag = ? AND model_name = ? AND tp_size = ? AND mtp_enabled = ? "
"AND ep_size IS ? AND dp_size IS ?",
(image_tag, _cfg.MODEL_NAME, tp_size, int(mtp_enabled), ep_size, dp_size),
).fetchone()

now = datetime.now(timezone.utc).isoformat()
Expand All @@ -273,8 +304,9 @@ def create_run(
cur = conn.execute(
"""INSERT INTO benchmark_runs
(image_tag, sglang_version, rocm_version, build_date,
model_name, tp_size, mtp_enabled, run_timestamp, status, result_dir)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
model_name, tp_size, mtp_enabled, ep_size, dp_size,
run_timestamp, status, result_dir)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
image_tag,
sglang_version,
Expand All @@ -283,6 +315,8 @@ def create_run(
_cfg.MODEL_NAME,
tp_size,
int(mtp_enabled),
ep_size,
dp_size,
now,
status,
result_dir,
Expand Down
50 changes: 43 additions & 7 deletions debug/perf-regression/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import os
import re
from pathlib import Path
from typing import NamedTuple, Optional

# ── Paths ──────────────────────────────────────────────────────────────────
BASE_DIR = Path(__file__).resolve().parent
Expand Down Expand Up @@ -72,18 +73,53 @@ def save_model_config(model_path: str, model_name: str) -> None:
MTP_MODE = True # Legacy: used by run_daily.sh; orchestrator uses TP_MTP_VARIANTS
CONCURRENCIES = "1,2,4"

# ── TP / MTP variant matrix ──────────────────────────────────────────────
# ── Variant matrix ───────────────────────────────────────────────────────


class VariantConfig(NamedTuple):
tp_size: int
mtp: bool
ep_size: Optional[int] = None
dp_size: Optional[int] = None


TP_MTP_VARIANTS = [
(4, False), # TP4
(4, True), # TP4+MTP
(8, False), # TP8
(8, True), # TP8+MTP
VariantConfig(4, False), # TP4
VariantConfig(4, True), # TP4+MTP
VariantConfig(8, False), # TP8
VariantConfig(8, True), # TP8+MTP
]


def variant_label(tp_size: int, mtp: bool) -> str:
def variant_label(tp_size: int, mtp: bool, ep_size: Optional[int] = None, dp_size: Optional[int] = None) -> str:
"""Human-readable variant label."""
return f"TP{tp_size}{'+MTP' if mtp else ''}"
label = f"TP{tp_size}{'+MTP' if mtp else ''}"
if ep_size is not None:
label += f"+EP{ep_size}"
if dp_size is not None:
label += f"+DP{dp_size}"
return label


_VARIANT_LABEL_RE = re.compile(
r"^TP(\d+)(\+MTP)?(?:\+EP(\d+))?(?:\+DP(\d+))?$", re.IGNORECASE
)


def parse_variant_label(label: str) -> Optional[VariantConfig]:
"""Parse a variant label like 'TP4+MTP+EP2+DP4' into a VariantConfig.

Returns None if the label doesn't match the expected format.
"""
m = _VARIANT_LABEL_RE.match(label)
if not m:
return None
return VariantConfig(
tp_size=int(m.group(1)),
mtp=m.group(2) is not None,
ep_size=int(m.group(3)) if m.group(3) else None,
dp_size=int(m.group(4)) if m.group(4) else None,
)


ACCURACY_MODE = True
Expand Down
85 changes: 76 additions & 9 deletions debug/perf-regression/dashboard.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ async def chart_data(
model_name: Optional[str] = Query(None),
tp_size: Optional[str] = Query(None),
mtp_enabled: Optional[str] = Query(None),
ep_size: Optional[str] = Query(None),
dp_size: Optional[str] = Query(None),
days: int = Query(30),
concurrency: Optional[str] = Query(None),
):
Expand All @@ -81,6 +83,20 @@ async def chart_data(
where_parts.append("br.mtp_enabled = ?")
params.append(int(mtp_enabled))

if ep_size and ep_size != "all":
if ep_size == "none":
where_parts.append("br.ep_size IS NULL")
else:
where_parts.append("br.ep_size = ?")
params.append(int(ep_size))

if dp_size and dp_size != "all":
if dp_size == "none":
where_parts.append("br.dp_size IS NULL")
else:
where_parts.append("br.dp_size = ?")
params.append(int(dp_size))

if days > 0:
where_parts.append(
f"br.build_date >= strftime('%Y%m%d', 'now', '-{days} days')"
Expand All @@ -105,6 +121,8 @@ async def chart_data(
br.sglang_version,
br.tp_size,
br.mtp_enabled,
br.ep_size,
br.dp_size,
bm.concurrency,
bm.output_throughput,
bm.total_throughput,
Expand All @@ -115,7 +133,7 @@ async def chart_data(
FROM benchmark_metrics bm
JOIN benchmark_runs br ON bm.run_id = br.id
WHERE {where_clause}{conc_filter}
ORDER BY br.build_date, br.rocm_version, br.tp_size, br.mtp_enabled, bm.concurrency
ORDER BY br.build_date, br.rocm_version, br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size, bm.concurrency
"""
rows = conn.execute(metrics_query, params_metrics).fetchall()

Expand All @@ -128,11 +146,13 @@ async def chart_data(
br.sglang_version,
br.tp_size,
br.mtp_enabled,
br.ep_size,
br.dp_size,
ar.accuracy_pct
FROM accuracy_results ar
JOIN benchmark_runs br ON ar.run_id = br.id
WHERE {where_clause}
ORDER BY br.build_date, br.rocm_version, br.tp_size, br.mtp_enabled
ORDER BY br.build_date, br.rocm_version, br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size
"""
accuracy_rows = conn.execute(accuracy_query, params).fetchall()

Expand Down Expand Up @@ -203,8 +223,14 @@ def _adjust_shade(hex_color: str, factor: float) -> str:
for row in rows:
mtp = row["mtp_enabled"]
tp = row["tp_size"]
ep = row["ep_size"]
dp = row["dp_size"]
mtp_label = "MTP" if mtp else "non-MTP"
key = f"rocm{row['rocm_version']}-tp{tp}-mtp{mtp}-c{row['concurrency']}"
ep_label = f"/EP{ep}" if ep is not None else ""
dp_label = f"/DP{dp}" if dp is not None else ""
ep_key = f"-ep{ep}" if ep is not None else ""
dp_key = f"-dp{dp}" if dp is not None else ""
key = f"rocm{row['rocm_version']}-tp{tp}-mtp{mtp}{ep_key}{dp_key}-c{row['concurrency']}"
conc = row["concurrency"]
color = rocm_tp_conc_colors.get(
(row["rocm_version"], tp, conc), "#607D8B"
Expand All @@ -220,7 +246,7 @@ def _adjust_shade(hex_color: str, factor: float) -> str:
ds = metric_charts[metric_name]["datasets"]
if key not in ds:
ds[key] = {
"label": f"ROCm {row['rocm_version']} / TP{tp} / {mtp_label} / c={conc}",
"label": f"ROCm {row['rocm_version']} / TP{tp} / {mtp_label}{ep_label}{dp_label} / c={conc}",
"data": [],
"borderColor": color,
"backgroundColor": color + "33",
Expand Down Expand Up @@ -262,16 +288,22 @@ def _adjust_shade(hex_color: str, factor: float) -> str:
for row in accuracy_rows:
mtp = row["mtp_enabled"]
tp = row["tp_size"]
ep = row["ep_size"]
dp = row["dp_size"]
mtp_label = "MTP" if mtp else "non-MTP"
key = f"rocm{row['rocm_version']}-tp{tp}-mtp{mtp}"
ep_label = f"/EP{ep}" if ep is not None else ""
dp_label = f"/DP{dp}" if dp is not None else ""
ep_key = f"-ep{ep}" if ep is not None else ""
dp_key = f"-dp{dp}" if dp is not None else ""
key = f"rocm{row['rocm_version']}-tp{tp}-mtp{mtp}{ep_key}{dp_key}"
color = rocm_tp_base.get(
(row["rocm_version"], tp), "#607D8B"
)
point_style = "rectRot" if mtp else "circle"

if key not in acc_datasets:
acc_datasets[key] = {
"label": f"ROCm {row['rocm_version']} / TP{tp} / {mtp_label}",
"label": f"ROCm {row['rocm_version']} / TP{tp} / {mtp_label}{ep_label}{dp_label}",
"data": [],
"borderColor": color,
"backgroundColor": color + "33",
Expand Down Expand Up @@ -316,6 +348,8 @@ async def get_runs(
model_name: Optional[str] = Query(None),
tp_size: Optional[str] = Query(None),
mtp_enabled: Optional[str] = Query(None),
ep_size: Optional[str] = Query(None),
dp_size: Optional[str] = Query(None),
days: int = Query(30),
limit: int = Query(100),
):
Expand All @@ -341,6 +375,20 @@ async def get_runs(
where_parts.append("br.mtp_enabled = ?")
params.append(int(mtp_enabled))

if ep_size and ep_size != "all":
if ep_size == "none":
where_parts.append("br.ep_size IS NULL")
else:
where_parts.append("br.ep_size = ?")
params.append(int(ep_size))

if dp_size and dp_size != "all":
if dp_size == "none":
where_parts.append("br.dp_size IS NULL")
else:
where_parts.append("br.dp_size = ?")
params.append(int(dp_size))

if days > 0:
where_parts.append(
f"br.build_date >= strftime('%Y%m%d', 'now', '-{days} days')"
Expand All @@ -353,6 +401,7 @@ async def get_runs(
SELECT
br.id, br.image_tag, br.sglang_version, br.rocm_version,
br.build_date, br.model_name, br.tp_size, br.mtp_enabled,
br.ep_size, br.dp_size,
br.run_timestamp, br.status,
br.error_message, br.duration_total_sec
FROM benchmark_runs br
Expand Down Expand Up @@ -456,6 +505,8 @@ async def get_alerts(
model_name: Optional[str] = Query(None),
tp_size: Optional[str] = Query(None),
mtp_enabled: Optional[str] = Query(None),
ep_size: Optional[str] = Query(None),
dp_size: Optional[str] = Query(None),
days: int = Query(30),
acknowledged: Optional[bool] = Query(None),
):
Expand All @@ -481,6 +532,20 @@ async def get_alerts(
where_parts.append("br.mtp_enabled = ?")
params.append(int(mtp_enabled))

if ep_size and ep_size != "all":
if ep_size == "none":
where_parts.append("br.ep_size IS NULL")
else:
where_parts.append("br.ep_size = ?")
params.append(int(ep_size))

if dp_size and dp_size != "all":
if dp_size == "none":
where_parts.append("br.dp_size IS NULL")
else:
where_parts.append("br.dp_size = ?")
params.append(int(dp_size))

if days > 0:
where_parts.append(
f"br.build_date >= strftime('%Y%m%d', 'now', '-{days} days')"
Expand All @@ -498,7 +563,7 @@ async def get_alerts(
ra.current_value, ra.baseline_value, ra.regression_pct,
ra.acknowledged, ra.created_at,
br.image_tag, br.build_date, br.rocm_version, br.sglang_version,
br.tp_size, br.mtp_enabled
br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size
FROM regression_alerts ra
JOIN benchmark_runs br ON ra.run_id = br.id
WHERE {where_clause}
Expand Down Expand Up @@ -527,9 +592,9 @@ async def variant_comparison(
runs = conn.execute(
f"""
SELECT br.id, br.model_name, br.image_tag, br.rocm_version,
br.tp_size, br.mtp_enabled
br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size
FROM benchmark_runs br WHERE {where}
ORDER BY br.tp_size, br.mtp_enabled
ORDER BY br.tp_size, br.mtp_enabled, br.ep_size, br.dp_size
""",
params,
).fetchall()
Expand All @@ -550,6 +615,8 @@ async def variant_comparison(
"rocm_version": run["rocm_version"],
"tp_size": run["tp_size"],
"mtp_enabled": run["mtp_enabled"],
"ep_size": run["ep_size"],
"dp_size": run["dp_size"],
"metrics": [dict(m) for m in metrics],
"accuracy_pct": accuracy["accuracy_pct"] if accuracy else None,
})
Expand Down
Loading
Loading