diff --git a/4.validation_and_observability/2.gpu-cluster-healthcheck/checks/6-efa-loopback.sh b/4.validation_and_observability/2.gpu-cluster-healthcheck/checks/6-efa-loopback.sh index 46c8b5e88..53afada41 100755 --- a/4.validation_and_observability/2.gpu-cluster-healthcheck/checks/6-efa-loopback.sh +++ b/4.validation_and_observability/2.gpu-cluster-healthcheck/checks/6-efa-loopback.sh @@ -21,13 +21,26 @@ EFA_INSTALLER_TEST="${EFA_INSTALLER_TEST:-/opt/amazon/efa/test/efa_test.sh}" # - FI_EFA_ENABLE_SHM_TRANSFER=0: force the real EFA hardware path; otherwise # libfabric routes same-host traffic through SHM and the test does not # exercise EFA at all. -# - FI_EFA_DEVICE_NAME=: pin libfabric to the specific EFA domain. +# - FI_EFA_IFACE=: pin libfabric to the specific EFA +# device. NOTE: this takes the kernel/ibv device name (e.g. "rdmap80s0"), +# not the libfabric domain name (e.g. "rdmap80s0-rdm") -- the "-rdm" suffix +# must be stripped from the domain string discovered below. +# (FI_EFA_DEVICE_NAME is NOT a real libfabric env var -- it does not +# appear in `fi_info -e`'s FI_EFA_* list and is silently ignored, which +# previously made every "per-device" iteration below run on whichever +# device libfabric picks by default. Confirmed against a running +# libfabric.so.1 2.4.0amzn3.0: 0 string occurrences of FI_EFA_DEVICE_NAME, +# vs. FI_EFA_IFACE present and honored.) # - explicit -B server_port / -B client_port -P server_port: avoid port # collisions when called per-device in a loop. # Returns 0 on success, non-zero on failure. Writes server+client logs to stdout # on failure for triage. run_pingpong_for_domain() { local domain="$1" + # FI_EFA_IFACE takes the kernel/ibv device name, not the libfabric domain + # name -- strip the "-rdm" suffix (see comment above run_pingpong_for_domain + # invocation site / the header comment block for why). + local iface="${domain%-rdm}" local server_port client_port server_port=$(shuf -n 1 -i 49152-57342) client_port=$(shuf -n 1 -i 57343-65535) @@ -36,7 +49,7 @@ run_pingpong_for_domain() { server_log=$(mktemp) client_log=$(mktemp) - FI_LOG_LEVEL=warn FI_EFA_ENABLE_SHM_TRANSFER=0 FI_EFA_DEVICE_NAME="${domain}" \ + FI_LOG_LEVEL=warn FI_EFA_ENABLE_SHM_TRANSFER=0 FI_EFA_IFACE="${iface}" \ fi_pingpong -e rdm -p efa -B "${server_port}" > "${server_log}" 2>&1 & local server_pid=$! sleep 3 @@ -50,7 +63,7 @@ run_pingpong_for_domain() { fi local ret=0 - FI_LOG_LEVEL=warn FI_EFA_ENABLE_SHM_TRANSFER=0 FI_EFA_DEVICE_NAME="${domain}" \ + FI_LOG_LEVEL=warn FI_EFA_ENABLE_SHM_TRANSFER=0 FI_EFA_IFACE="${iface}" \ timeout "${EFA_TEST_TIMEOUT}" \ fi_pingpong -e rdm -p efa -B "${client_port}" -P "${server_port}" localhost \ > "${client_log}" 2>&1 || ret=$? @@ -99,11 +112,11 @@ run_check() { # Discover EFA libfabric DOMAINS, not kernel ibv device names. The two # naming spaces differ: ibv_devices returns names like 'rdmap86s0', but - # libfabric's -d/FI_EFA_DEVICE_NAME expects domains like 'rdmap86s0-rdm' - # (with the '-rdm' suffix added by the EFA provider). Passing kernel names - # to fi_pingpong yields fi_getinfo -61 (No data available) and the test - # fails on every device. Enumerating via fi_info gets us the correct names - # and also naturally excludes back-side Ethernet NICs that show up under + # libfabric domains (as reported by `fi_info -p efa -t FI_EP_RDM`) are + # named like 'rdmap86s0-rdm'. FI_EFA_IFACE (used to pin below) takes the + # kernel/ibv name, so the '-rdm' suffix gets stripped per-device. + # Enumerating via fi_info gets us the correct domain names and also + # naturally excludes back-side Ethernet NICs that show up under # ibv_devices but are not EFA endpoints. local domains domains=$(fi_info -p efa -t FI_EP_RDM 2>/dev/null \ @@ -119,6 +132,25 @@ run_check() { device_count=$(echo "${domains}" | wc -l | tr -d ' ') log_info "Testing ${device_count} EFA domain(s)" + # Regression guard: a bogus, definitely-nonexistent device name MUST fail. + # This is the exact failure signature of the FI_EFA_DEVICE_NAME defect + # (a per-device pinning env var that libfabric silently ignores, so every + # "per-device" iteration -- including one given a nonexistent name -- ran + # on whichever device libfabric picked by default and reported PASS). If + # this negative control ever passes, per-device pinning is broken again + # and the results below cannot be trusted as per-device -- fail loudly + # instead of reporting a green per-device sweep that isn't one. + log_verbose "Running negative-control check: bogus device name must fail loopback" + local negctrl_exit=0 + run_pingpong_for_domain "definitely-nonexistent-device-rdm" > /dev/null 2>&1 || negctrl_exit=$? + if [[ "${negctrl_exit}" -eq 0 ]]; then + check_fail "${CHECK_NAME}" \ + "Negative control failed: a nonexistent device name (definitely-nonexistent-device-rdm) returned PASS. Per-device pinning is not working -- results below cannot be trusted as per-device. (This is the FI_EFA_DEVICE_NAME silent-no-op failure mode; see fix commit.)" \ + "RESET" + return 1 + fi + log_verbose "Negative control OK: bogus device name correctly failed (exit ${negctrl_exit})" + local failures=0 local results_json="[" diff --git a/4.validation_and_observability/2.gpu-cluster-healthcheck/kubernetes/agent.sh b/4.validation_and_observability/2.gpu-cluster-healthcheck/kubernetes/agent.sh index c9991e579..60348c809 100755 --- a/4.validation_and_observability/2.gpu-cluster-healthcheck/kubernetes/agent.sh +++ b/4.validation_and_observability/2.gpu-cluster-healthcheck/kubernetes/agent.sh @@ -51,17 +51,17 @@ require_env() { fi } -# Detect instance type: IMDS first, then K8s node label fallback +# Detect instance type: IMDS first, then K8s node label fallback. +# NOTE: detect_instance_type() in lib/common.sh already implements this exact +# fallback chain (IMDS -> ec2-metadata -> kubectl node-label, gated on +# NODE_NAME being set) as of the common.sh fix for the check-2 vacuous-PASS +# defect -- this wrapper now just adds agent.sh's own "unknown" default +# instead of reinventing the kubectl fallback here. NODE_NAME is required by +# main()'s require_env call before this runs, so the k8s-label branch inside +# detect_instance_type() is live for this caller. detect_instance() { - # Try IMDS (works with hostNetwork: true) INSTANCE_TYPE=$(detect_instance_type 2>/dev/null || true) - if [[ -z "${INSTANCE_TYPE}" ]]; then - log_info "IMDS unavailable, falling back to node label" - INSTANCE_TYPE=$(kubectl get node "${NODE_NAME}" \ - -o jsonpath='{.metadata.labels.node\.kubernetes\.io/instance-type}' 2>/dev/null || true) - fi - if [[ -z "${INSTANCE_TYPE}" ]]; then log_warn "Unable to detect instance type -- using defaults" INSTANCE_TYPE="unknown" diff --git a/4.validation_and_observability/2.gpu-cluster-healthcheck/kubernetes/manifests/05-job-quarantine.yaml b/4.validation_and_observability/2.gpu-cluster-healthcheck/kubernetes/manifests/05-job-quarantine.yaml index 38881b9d5..40d883ed6 100644 --- a/4.validation_and_observability/2.gpu-cluster-healthcheck/kubernetes/manifests/05-job-quarantine.yaml +++ b/4.validation_and_observability/2.gpu-cluster-healthcheck/kubernetes/manifests/05-job-quarantine.yaml @@ -60,6 +60,12 @@ spec: LABEL_PREFIX="gpu-healthcheck.aws-samples.io" RESULTS_DIR="/tmp/gpu-healthcheck-quarantine" export RESULTS_DIR + # NODE_NAME is lib/common.sh's detect_instance_type() kubectl + # fallback key (used when IMDS is unreachable, which it is by + # default from an EKS pod). This Job's ServiceAccount already + # has `get` on nodes (02-rbac.yaml), so exporting this here is + # sufficient -- no RBAC or volume changes needed. + export NODE_NAME="${NODE}" echo "=== GPU Quarantine Check: ${NODE} ===" diff --git a/4.validation_and_observability/2.gpu-cluster-healthcheck/lib/common.sh b/4.validation_and_observability/2.gpu-cluster-healthcheck/lib/common.sh index 805a28f6d..d67cc3434 100644 --- a/4.validation_and_observability/2.gpu-cluster-healthcheck/lib/common.sh +++ b/4.validation_and_observability/2.gpu-cluster-healthcheck/lib/common.sh @@ -26,7 +26,11 @@ DRY_RUN="${DRY_RUN:-0}" JSON_OUTPUT="${JSON_OUTPUT:-0}" # Instance profile variables (populated by load_instance_profile) -INSTANCE_TYPE="" +# INSTANCE_TYPE may be pre-exported by the caller (e.g. a k8s manifest +# threading the node's `node.kubernetes.io/instance-type` label down as an +# env var) to bypass IMDS detection entirely -- see detect_instance_type() +# below for why IMDS alone is not sufficient inside an EKS pod. +INSTANCE_TYPE="${INSTANCE_TYPE:-}" EXPECTED_GPU_COUNT="" EXPECTED_EFA_COUNT="" NVLINK_EXPECTED="" @@ -54,6 +58,13 @@ log_verbose() { # ─── Instance Detection ───────────────────────────────────────────────────── +# IMDS is unreachable from a pod running inside EKS under default hop-limit +# settings (confirmed: IMDSv2 token request returns empty, IMDSv1 fallback +# also empty). load_instance_profile() below only calls this when +# INSTANCE_TYPE is not already set -- callers that know their instance type +# some other way (e.g. a k8s Downward API / node-label lookup) should export +# INSTANCE_TYPE before sourcing this file, or before calling +# load_instance_profile(), to skip IMDS entirely. detect_instance_type() { # Try IMDSv2 first, fall back to IMDSv1, then ec2-metadata CLI local token @@ -73,8 +84,20 @@ detect_instance_type() { INSTANCE_TYPE=$(ec2-metadata --instance-type 2>/dev/null | awk '{print $2}' || true) fi + # Last resort: k8s node label, when running as a pod with NODE_NAME set + # (downward API `spec.nodeName`) and `kubectl` reachable. This mirrors + # kubernetes/agent.sh's own IMDS-unavailable fallback (agent.sh:59-63) -- + # centralized here so any caller of load_instance_profile() gets it, not + # just the DaemonSet agent. Requires the ServiceAccount to have `get` on + # `nodes` (already granted: kubernetes/manifests/02-rbac.yaml:19-20). + if [[ -z "${INSTANCE_TYPE}" && -n "${NODE_NAME:-}" ]] && command -v kubectl > /dev/null 2>&1; then + log_warn "IMDS unavailable -- falling back to k8s node label for instance type" + INSTANCE_TYPE=$(kubectl get node "${NODE_NAME}" \ + -o jsonpath='{.metadata.labels.node\.kubernetes\.io/instance-type}' 2>/dev/null || true) + fi + if [[ -z "${INSTANCE_TYPE}" ]]; then - log_error "Unable to detect instance type via IMDS or ec2-metadata" + log_error "Unable to detect instance type via IMDS, ec2-metadata, or k8s node label" return 1 fi