diff --git a/docs/contributor/validator.md b/docs/contributor/validator.md index 78d1a2973..66459e56b 100644 --- a/docs/contributor/validator.md +++ b/docs/contributor/validator.md @@ -692,7 +692,8 @@ silently fall back. Full list (defaults, semantics) is in the `validators/performance` package godoc. NCCL variants exposed today: `nccl-all-reduce-bw`, -`nccl-all-reduce-bw-net`, `nccl-all-reduce-bw-nvls`. Inference: +`nccl-all-reduce-bw-net`, `nccl-all-reduce-bw-nvls`. Opt-in CRE NCCL +for EKS H100: `nccl-cre-all-reduce-bw`. Inference: `inference-perf` (Dynamo + AIPerf). > **Constraint-name contract.** Each NCCL variant looks up a diff --git a/docs/user/validation.md b/docs/user/validation.md index bed2f8405..52fae8031 100644 --- a/docs/user/validation.md +++ b/docs/user/validation.md @@ -53,6 +53,15 @@ ones) that match the target fabric: | `nccl-all-reduce-bw-net` | NET (EFA on EKS by default; ConnectX RoCE via `AICR_NCCL_FABRIC=roce`) | GB200 + EKS. Asserts EFA actually carried traffic — catches silent fallback to Socket when the NVIDIA driver is missing `NVreg_GrdmaPciTopoCheckOverride=1`. | | `nccl-all-reduce-bw-nvls` | NVLS (MNNVL across an NVL72 IMEX domain) | GB200 + EKS, and GB200 + OKE. Asserts the NVLS communicator actually initialized — catches silent fallback to EFA (EKS) or Socket (OKE) when the IMEX domain is misconfigured. | +An opt-in Cluster Readiness Engine (CRE) NCCL check is available for EKS H100. +It is not attached to shipped overlays while CRE remains private and the +result has not been correlated with the TrainJob path. The check requires a +same-named constraint: + +| Check | What it measures | +|---|---| +| `nccl-cre-all-reduce-bw` | EFA bus bandwidth from a CRE `WorkloadRun` `BandwidthMeasurement`; AICR still asserts the transport from launcher logs | + The applicability column is the *default*, derived from the recipe's `criteria`. A recipe whose criteria fall outside it can still run these benchmarks explicitly — either by diff --git a/pkg/validator/catalog/catalog_test.go b/pkg/validator/catalog/catalog_test.go index 478f05223..298c37a56 100644 --- a/pkg/validator/catalog/catalog_test.go +++ b/pkg/validator/catalog/catalog_test.go @@ -1162,6 +1162,22 @@ func TestEmbeddedCatalog_NCCLAllReduceBWNetEntryExists(t *testing.T) { t.Fatalf("no embedded catalog entry named %q (AICR_NCCL_FABRIC forwarding would silently no-op)", v1.NCCLAllReduceBWNetCheckName) } +func TestEmbeddedCatalog_CRENCCLAllReduceBWEntryExists(t *testing.T) { + cat, err := LoadWithDataProvider(context.Background(), nil, "v0.0.0-next", "") + if err != nil { + t.Fatalf("Load failed: %v", err) + } + for _, v := range cat.Validators { + if v.Name == v1.CRENCCLAllReduceBWCheckName { + if v.Phase != "performance" { + t.Errorf("%q phase = %q, want performance", v1.CRENCCLAllReduceBWCheckName, v.Phase) + } + return + } + } + t.Fatalf("no embedded catalog entry named %q", v1.CRENCCLAllReduceBWCheckName) +} + func TestCatalogEmbedding(t *testing.T) { // Simulate embedding in a CR spec type ValidatorCatalogSpec struct { diff --git a/pkg/validator/v1/job_plan_internal.go b/pkg/validator/v1/job_plan_internal.go index b86678277..274e38e4b 100644 --- a/pkg/validator/v1/job_plan_internal.go +++ b/pkg/validator/v1/job_plan_internal.go @@ -59,6 +59,12 @@ const ( // would otherwise silently no-op RoCE forwarding with no test failing. NCCLAllReduceBWNetCheckName = "nccl-all-reduce-bw-net" + // CRENCCLAllReduceBWCheckName is the catalog name of the CRE-driven NCCL + // all-reduce check (WorkloadRun). Overlays opt in by listing this check and + // a same-named performance constraint. It is limited to EKS H100 while CRE + // integration is validated, and no embedded overlay enables it. + CRENCCLAllReduceBWCheckName = "nccl-cre-all-reduce-bw" + // ncclFabricEnv selects the NET fabric (efa default | roce). Forwarded to // the NET check pod so the in-Job validator can observe it. This is the // orchestrator (forwarding) end; the validator-pod (reading) end defines the diff --git a/recipes/validators/catalog.yaml b/recipes/validators/catalog.yaml index 956267042..448241274 100644 --- a/recipes/validators/catalog.yaml +++ b/recipes/validators/catalog.yaml @@ -286,3 +286,13 @@ validators: timeout: 30m args: ["nccl-all-reduce-bw-nvls"] env: [] + # CRE-driven EKS H100 NCCL check. Opt-in only: no embedded overlay + # references this check while CRE remains private and correlation with + # the TrainJob path is incomplete. + - name: nccl-cre-all-reduce-bw + phase: performance + description: "Verify EKS H100 NCCL All Reduce Bus Bandwidth via a Cluster Readiness Engine WorkloadRun" + image: ghcr.io/nvidia/aicr-validators/performance:latest + timeout: 30m + args: ["nccl-cre-all-reduce-bw"] + env: []