diff --git a/presto/scripts/perf_flamegraph.py b/presto/scripts/perf_flamegraph.py new file mode 100755 index 00000000..2196ffdb --- /dev/null +++ b/presto/scripts/perf_flamegraph.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Render perf-script call chains as folded stacks and interactive HTML. + +This is the dependency-free fallback for hosts whose perf package omits the +in-tree ``perf script report flamegraph`` Python reporter. It does not use the +legacy Perl FlameGraph stack-collapse or rendering scripts. The generated +self-contained flamegraph supports click-to-zoom without loading JavaScript +or CSS from a CDN. +""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import re +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +FRAME_RE = re.compile(r"^([0-9a-fA-F]+)\s+(.+?)\s+\((.*)\)\s*$") +OFFSET_RE = re.compile(r"\+0x[0-9a-fA-F]+(?:/0x[0-9a-fA-F]+)?$") + + +def clean_symbol(symbol: str, dso: str) -> str: + """Normalize a frame so samples at different instruction offsets merge.""" + + symbol = OFFSET_RE.sub("", symbol.strip()).replace(";", ":") + if not symbol or symbol in {"[unknown]", "unknown"}: + dso_name = Path(dso).name if dso and dso != "[unknown]" else "unknown" + return f"[unknown:{dso_name}]" + return symbol + + +def collapse_perf_script(path: Path) -> Counter[tuple[str, ...]]: + """Return root-to-leaf stacks and their sample counts.""" + + stacks: Counter[tuple[str, ...]] = Counter() + frames: list[str] = [] + + def flush() -> None: + nonlocal frames + if frames: + # perf script emits the sampled leaf first and its callers after it. + stacks[tuple(reversed(frames))] += 1 + frames = [] + + with path.open(errors="replace") as source: + for raw_line in source: + if not raw_line.strip(): + flush() + continue + if raw_line.startswith("#"): + continue + if not raw_line[0].isspace(): + # Some perf versions omit the blank separator between samples. + flush() + continue + + match = FRAME_RE.match(raw_line.strip()) + if match: + _address, symbol, dso = match.groups() + frames.append(clean_symbol(symbol, dso)) + flush() + + return stacks + + +def read_folded(path: Path) -> Counter[tuple[str, ...]]: + """Read an existing folded-stack file for fast HTML-only rerendering.""" + + stacks: Counter[tuple[str, ...]] = Counter() + with path.open(errors="replace") as source: + for line_number, raw_line in enumerate(source, start=1): + line = raw_line.rstrip("\n") + if not line: + continue + try: + stack_text, count_text = line.rsplit(maxsplit=1) + count = int(count_text) + except (ValueError, TypeError) as error: + raise ValueError(f"{path}:{line_number}: invalid folded-stack record") from error + stack = tuple(stack_text.split(";")) + if count <= 0 or not stack or any(not frame for frame in stack): + raise ValueError(f"{path}:{line_number}: invalid folded-stack record") + stacks[stack] += count + return stacks + + +@dataclass +class Node: + name: str + count: int = 0 + children: dict[str, "Node"] = field(default_factory=dict) + + +def build_tree(stacks: Counter[tuple[str, ...]]) -> tuple[Node, int]: + root = Node("all") + max_depth = 0 + for stack, count in stacks.items(): + root.count += count + node = root + max_depth = max(max_depth, len(stack)) + for frame in stack: + node = node.children.setdefault(frame, Node(frame)) + node.count += count + return root, max_depth + + +def frame_color(name: str) -> str: + digest = hashlib.sha256(name.encode(errors="replace")).digest() + return f"rgb({205 + digest[0] % 45},{75 + digest[1] % 105},{35 + digest[2] % 55})" + + +def shorten(text: str, width: float) -> str: + max_chars = max(0, int((width - 6) / 7.0)) + if max_chars < 3: + return "" + return text if len(text) <= max_chars else text[: max_chars - 2] + ".." + + +def render_html(root: Node, max_depth: int, destination: Path, title: str) -> None: + if root.count <= 0: + raise ValueError("perf script contained no call-chain samples") + + width = 1600 + frame_height = 18 + left = 12 + right = 12 + header = 78 + bottom = 24 + plot_width = width - left - right + height = header + max(1, max_depth) * frame_height + bottom + scale = plot_width / root.count + elements: list[str] = [] + + def draw_children(node: Node, x: float, depth: int, parent_path: str) -> None: + cursor = x + for child_index, child in enumerate(sorted(node.children.values(), key=lambda item: item.name)): + child_width = child.count * scale + y = height - bottom - (depth + 1) * frame_height + node_path = f"{parent_path}/{child_index}" if parent_path else str(child_index) + name_attr = html.escape(child.name, quote=True) + percentage = 100.0 * child.count / root.count + tooltip = html.escape(f"{child.name} — {child.count} samples ({percentage:.2f}%)", quote=True) + if child_width >= 0.3: + elements.append( + f'' + f"{tooltip}" + f'' + f'' + f"{html.escape(shorten(child.name, child_width))}" + ) + draw_children(child, cursor, depth + 1, node_path) + cursor += child_width + + draw_children(root, left, 0, "") + escaped_title = html.escape(title) + document = f""" + + + +{escaped_title} + + + +

{escaped_title}

+ + +Showing all frames +

{root.count} sampled call chains. Click a frame to zoom; hover for sample count and percentage.

+ +{"".join(elements)} + + + + +""" + destination.write_text(document, encoding="utf-8") + + +def write_folded(stacks: Counter[tuple[str, ...]], destination: Path) -> None: + with destination.open("w") as output: + for stack, count in sorted(stacks.items(), key=lambda item: item[0]): + output.write(f"{';'.join(stack)} {count}\n") + + +def known_sample_count(stacks: Counter[tuple[str, ...]]) -> int: + """Count samples containing at least one symbolized frame.""" + + return sum(count for stack, count in stacks.items() if any(not frame.startswith("[unknown:") for frame in stack)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--input", type=Path, help="perf script text") + source.add_argument( + "--input-folded", + type=Path, + help="existing folded stacks for fast HTML-only rerendering", + ) + parser.add_argument( + "--folded", + type=Path, + help="folded-stack output (required with --input)", + ) + parser.add_argument("--output", type=Path, required=True, help="self-contained HTML output") + parser.add_argument("--title", default="perf flamegraph") + parser.add_argument( + "--min-known-sample-ratio", + type=float, + default=0.05, + help="minimum fraction of call chains with a symbolized frame (default: 0.05)", + ) + args = parser.parse_args() + + if not 0.0 <= args.min_known_sample_ratio <= 1.0: + parser.error("--min-known-sample-ratio must be between 0 and 1") + if args.input is not None and args.folded is None: + parser.error("--folded is required with --input") + if args.input_folded is not None and args.folded is not None: + parser.error("--folded cannot be used with --input-folded") + + try: + stacks = collapse_perf_script(args.input) if args.input is not None else read_folded(args.input_folded) + except ValueError as error: + parser.error(str(error)) + if not stacks: + parser.error("no call-chain samples found in input") + total_samples = sum(stacks.values()) + known_samples = known_sample_count(stacks) + known_ratio = known_samples / total_samples + if known_ratio < args.min_known_sample_ratio: + parser.error( + "only " + f"{known_samples}/{total_samples} call chains ({known_ratio:.2%}) " + "contain a symbolized frame; retain perf.data/perf.script.gz and " + "use the captured symfs or a matching symbol build" + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + if args.folded is not None: + write_folded(stacks, args.folded) + root, max_depth = build_tree(stacks) + render_html(root, max_depth, args.output, args.title) + print( + f"samples={root.count} known_samples={known_samples} " + f"known_ratio={known_ratio:.6f} unique_stacks={len(stacks)} " + f"max_depth={max_depth}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/presto/scripts/summarize_coordinator_queries.py b/presto/scripts/summarize_coordinator_queries.py new file mode 100755 index 00000000..d8d026b8 --- /dev/null +++ b/presto/scripts/summarize_coordinator_queries.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Summarize coordinator query JSON files into a TSV report. + +Usage: + summarize_coordinator_queries.py + +Reads /queries/*.json (one file per query) and writes: + /summary.tsv — tab-separated summary of all queries + /operator_summaries/ — per-query operatorSummaries JSON (if present) +""" + +import json +import pathlib +import sys + +if len(sys.argv) < 2: + sys.exit("Usage: summarize_coordinator_queries.py ") + +out = pathlib.Path(sys.argv[1]) +summary_rows = [ + [ + "query_id", + "state", + "error_type", + "error_name", + "elapsed", + "execution", + "query_preview", + ] +] + +for path in sorted((out / "queries").glob("*.json")): + try: + with path.open() as f: + query = json.load(f) + except json.JSONDecodeError as e: + print(f"Warning: skipping {path.name}: {e}", file=sys.stderr) + continue + + stats = query.get("queryStats") or {} + error_code = query.get("errorCode") or {} + sql = (query.get("query") or "").replace("\n", " ") + if len(sql) > 180: + sql = sql[:177] + "..." + + summary_rows.append( + [ + query.get("queryId", path.stem), + query.get("state", ""), + query.get("errorType", ""), + error_code.get("name", ""), + stats.get("elapsedTime", ""), + stats.get("executionTime", ""), + sql, + ] + ) + + operators = stats.get("operatorSummaries") or [] + if operators: + operators_dir = out / "operator_summaries" + operators_dir.mkdir(exist_ok=True) + with (operators_dir / f"{query.get('queryId', path.stem)}.operators.json").open("w") as f: + json.dump(operators, f, indent=2) + +with (out / "summary.tsv").open("w") as f: + for row in summary_rows: + f.write("\t".join(str(value) for value in row) + "\n") diff --git a/presto/slurm/presto-nvl72/README.md b/presto/slurm/presto-nvl72/README.md index 39cdc553..286632df 100644 --- a/presto/slurm/presto-nvl72/README.md +++ b/presto/slurm/presto-nvl72/README.md @@ -105,18 +105,276 @@ This step only needs to repeat when the worker image's stat tracking changes ``` Submits a benchmark sbatch job, polls until completion, and prints a summary. -Results land under `result_dir/`. See `./launch-run.sh --help` for the full -flag list (queries filter, output path, GDS toggle, profiling, metrics, …). +Results land under an immutable `result_dir_/` directory. A later launch +does not delete or write into it, so it is safe to `scp` while another benchmark +runs. `latest_result_dir.txt` contains the newest job-specific directory name; +resolve that small pointer once before a copy rather than copying a stale legacy +`result_dir/` or following a mutable symlink. See `./launch-run.sh --help` for +the full flag list (queries filter, output path, GDS toggle, profiling, +metrics, …). **Prerequisites:** worker + coord images on disk; data from step 1; analyzed metastore from step 2 (either local or shared). +### CPU and GPU NUMA placement + +`CLUSTER_{CPU,GPU}_USE_NUMA` is a Boolean switch; `0` means unbound and does +not mean NUMA node 0. CPU jobs discover CPU-bearing DRAM nodes on every +allocated host and ignore accelerator/HBM-only NUMA IDs. Local worker slots are +balanced in contiguous groups. On the two-socket, four-GPU GB200 topology this +gives: + +| Workers per host (`-g`) | CPU worker mapping | +|---:|---| +| 1 | `0` | +| 2 | `0,1` | +| 4 | `0,0,1,1` | + +Each CPU worker is bound with both `--cpunodebind` and `--membind`; its driver +and memory defaults are sized from its share of that CPU/DRAM node rather than +from host `MemTotal`, which also includes the four HBM proximity domains. +Every selected host must report the same CPU topology. Each worker log records +`numactl --show` plus `Cpus_allowed_list` and `Mems_allowed_list` after entering +the requested policy and before `exec`-ing `presto_server`. + +GPU jobs retain their explicit `CLUSTER_GPU_NUMA_GPUS_PER_NODE` mapping. For +the topology above, setting it to `2` binds GPU workers 0–1 to CPU node 0 and +GPU workers 2–3 to CPU node 1. Values such as `2,10,18,26` in the +`nvidia-smi topo -m` **GPU NUMA ID** column are HBM nodes and are not valid CPU +binding targets. + +For a one-worker historical comparison only, `--legacy-cpu-numa` reproduces +the old inconsistent behavior: it uses the deprecated +`numactl --cpubind=0 --membind=0` spelling while retaining configuration sized +for the complete host. On this topology the process still has only CPUs 0–71; +`task.max-drivers-per-task=144` oversubscribes those 72 cores rather than +granting 144 cores. The option requires `--cpu -g 1`, emits a warning, and +records the effective policy in the worker log. It is not a supported scaling +configuration. + +Before startup, the job verifies that the coordinator, worker HTTP, and +exchange ports are free on each host. On exit it terminates only the exact +coordinator/worker `srun` clients it launched, waits for their steps, and +verifies port release. Listener ownership, command lines, and Slurm cgroups +are retained in the result logs when startup or cleanup fails. +Exchange ports are tested with an actual wildcard bind as well as `ss`, so a +closed UCX listener that is no longer in `LISTEN` but is still not reusable is +not mistaken for a free port. Preflight and teardown wait up to 90 seconds by +default; override `PRESTO_PORT_PREFLIGHT_TIMEOUT` or +`PRESTO_PORT_RELEASE_TIMEOUT` when needed. + +For a CPU UCX validation run, `--verify-cpu-ucx` waits up to 60 seconds for +every generated HTTP+3 exchange port to be claimed before queries begin. For a +multi-worker run, it then requires the retained query details to contain a +`UcxCpuRowExchange` or `UcxCpuRowPartitionedOutput` operator. A one-worker run +checks listener readiness but skips that runtime operator assertion because it +cannot prove inter-worker transport. The readiness check does not depend on UCX +log-message spelling or timing. The option is CLI-only: an inherited +`VERIFY_CPU_UCX` environment variable cannot enable it. Use an exchange-heavy +query with at least two workers for end-to-end transport verification; +`PRESTO_CPU_UCX_READY_TIMEOUT` changes the readiness timeout. + ### Override images for a single run ```bash ./launch-run.sh -n 2 -s 100 -w my-worker-image -c my-coord-image ``` +### Query-scoped CPU flamegraph + +`--perf` attaches the compute host's kernel-matched `perf` to worker 0. It does +not require `perf` inside the Enroot image. Keep this separate from `-p/--profile` +(nsys), and select one representative query and iteration: + +```bash +./launch-run.sh --cpu -n 10 -g 1 -s 1000 -q 18 -i 2 \ + --perf --profile-iterations 1 \ + -w -c +``` + +The sidecar preserves or raises its inherited host `nofile` limit to the hard +limit granted to the Slurm job, then tests the requested event against the live +worker. For each capture, it snapshots every live worker TID and partitions +them among bounded `perf record` processes. Every recorder starts with events +disabled. Recorders are initialized serially and must answer a control `ping` +before the query-side enable barrier; a transient initialization failure is +retried up to three times. The query starts only after every recorder +acknowledges that every target event is open. Inheritance covers worker threads +created after that snapshot. An unrecoverable attach-time descriptor or setup +failure therefore aborts before the measured query starts. Missing host `perf`, +a low hard descriptor limit, +`perf_event_paranoid`, PMU, ptrace, or Slurm cgroup restrictions likewise fail +the run explicitly instead of producing an empty profile. Capture metadata +records the live worker thread/mapping counts, shard membership, and each +recorder's open descriptor count for capacity diagnosis. Every active raw +capture is retained on node-local +`${SLURM_TMPDIR:-/tmp}` (`PERF_CAPTURE_TMPDIR` overrides the staging base). +Stopping a query waits only until every recorder has stopped, closed its output, +and queued those node-local files. This prevents home/NFS publication and +symbolization from blocking pytest or perturbing the next measured query. + +After the complete query suite, the sidecar atomically publishes all queued +`perf.data` shards into the result directory before worker teardown. It also +snapshots the worker's executable mappings into a portable `symfs/`; this keeps +symbolization independent of Enroot's private mount namespace and permits +reprocessing after the job. Publication progress and byte counts are written to +`logs/perf_worker_0.log`. By default, expensive derived-report generation is +deferred: + +```text +result_dir_/profiles/perf/worker_0/ +├── symfs/ # exact mapped executables/shared libraries +├── symbol-manifest.tsv +└── Q18_iter1/ + ├── perf.data.part000 + ├── perf.data.part001 + ├── perf-shards.tsv + ├── perf-startup-attempts.tsv + ├── perf-target-tids.txt + ├── perf-attached-tids.part000.txt + ├── perf-dropped-tids.tsv + └── capture-metadata.txt +``` + +One-shard captures retain the conventional `perf.data` name. For a multi-shard +capture, `process-perf-captures.sh` symbolizes each raw file and concatenates +the sample streams into one `perf.script.gz` and one aggregate flamegraph. The +text report labels each raw shard because its percentages are shard-local. +The generated `flamegraph.html` is self-contained: click a frame to zoom into +that subtree, use **Reset Zoom** to return, and use the search box to highlight +matching functions. + +The recorder snapshots disjoint worker TID sets, but worker-pool threads can +exit while the shards are being initialized. Before every startup attempt, it +filters each shard against `/proc//task`; an exited TID is recorded in +`perf-dropped-tids.tsv` instead of making every retry fail with `ESRCH`. The +per-shard `perf-attached-tids.*.txt` files preserve the exact final target set. +Inheritance remains enabled for threads created after attachment. + +Generate `perf-report.txt`, `perf.script.gz`, `stacks.folded`, and +`flamegraph.html` later from the login/head node with: + +```bash +./process-perf-captures.sh result_dir_ + +# Or process one capture: +./process-perf-captures.sh result_dir_ --query Q18_iter1 + +# Omit inline-function expansion for a much faster, still symbolized graph: +./process-perf-captures.sh result_dir_ \ + --query Q18_iter1 --no-inline +``` + +An already generated aggregate `stacks.folded` is architecture-independent. +To upgrade or regenerate only its interactive HTML on the head node—without +rerunning `perf script` or requesting a compute allocation—run: + +```bash +capture=result_dir_JOBID/profiles/perf/worker_0/Q18_iter1 +python3 ../../scripts/perf_flamegraph.py \ + --input-folded "${capture}/stacks.folded" \ + --output "${capture}/flamegraph.html" \ + --title "Presto worker 0: Q18_iter1" +``` + +When called outside Slurm, the script uses the CPU partition/account from +`~/.cluster_config.env` and starts one blocking `srun` compute allocation. No +host name or rail is selected or hard-coded. It requests one CPU per raw +recorder shard, copies the capture and `symfs/` to +`${SLURM_TMPDIR:-/tmp}`, symbolizes the shards in parallel, renders the combined +stream, and atomically copies only the derived artifacts back. This ensures an +Arm64 Grace capture is unwound by an Arm64 `perf`, avoids repeated random I/O +against the shared result directory, and prevents copy permission warnings by +not preserving shared-filesystem metadata in the temporary tree. `--jobs N` +caps shard concurrency, `--time HH:MM:SS` changes the processing allocation, +and `--local` bypasses `srun` when the command is already running on a +compatible compute host. + +`--no-inline` is passed to both `perf report` and `perf script`, including +across the automatic `srun` relaunch. It disables inline-function expansion, +not DWARF stack unwinding: physical call chains, symbols, raw data, and the +portable `symfs/` remain intact. The chosen mode, compute hostname, +architecture, shard count, and parallelism are recorded in +`postprocess-metadata.txt`. Re-running with a different inline mode regenerates +the derived files automatically; `--force` is only required to repeat the same +mode. + +Pass `--perf-postprocess` to the original launch only when those derived files +must be generated inside the allocation. That work still waits until all +queries and raw-file publication have completed, so it cannot bias later query +captures. The postprocessor uses the bundled dependency-free Python renderer +over the combined `perf script` output, so it needs neither a network download +nor the old Perl FlameGraph scripts. + +The bundled renderer rejects output when fewer than 5% of sampled call chains +contain a known symbol, rather than calling an all-`[unknown]` graph successful. +It also refuses any raw directory whose metadata lacks the complete +ready/enable/stop/publish sequence, whose shard manifest contains a failed +recorder, or whose published shard count is incomplete. Diagnostic partial +logs and metadata from a failed attach are retained, while its unusable +pre-query partial `perf.data` files are discarded instead of being copied to +shared storage. A failed attach therefore cannot silently become a folded stack +file or flamegraph. +If rendering fails, the command exits nonzero and publishes +`flamegraph-unavailable.txt`, `perf.script.gz`, the text report, and diagnostics; +the raw `perf.data` shards and matching `symfs/` are never removed or modified. + +Profiled benchmark runs stop after the first failed query, preventing a broken +sampler setup from wasting the remainder of a full suite. + +Use `--perf-frequency`, `--perf-event`, `--perf-call-graph`, +`--perf-user-regs`, `--perf-nofile-limit`, and +`--perf-threads-per-shard` to override the defaults (`19`, `cpu-clock:u`, +`dwarf,8192`, `auto`, a `65536` minimum, and `128` target TIDs per recorder). +The sidecar never lowers a larger inherited soft limit and raises it to the +job's hard limit when permitted. `perf record` gets 120 seconds to flush and +stop after each query; `--perf-record-stop-timeout` overrides that watchdog. +These conservative +sampling defaults are intentional on a 144-core worker: higher-frequency DWARF +capture can lose samples, create hundreds of megabytes per short query, and +materially perturb the query being measured. Profile-run timings should not be +posted as benchmark timings. If `-o/--output-path` is given, the profile is +included in the normal result-directory copy. + +`cpu-clock:u` is intentionally a software, user-space, on-CPU sampling event. +It produces the CPU-time flamegraph needed for hotspot analysis without relying +on a particular hardware counter. On this Grace/NVIDIA Linux 6.14 stack, perf +can still expand an event across the PMU CPU map for every target thread. A +warmed Presto worker has over one thousand threads, so one recorder can exceed +the job's 131072-descriptor hard limit. The TID shards bound descriptors per +recorder without dropping threads or restricting the CPUs on which they may be +sampled. `--perf-event cycles:u` remains available for hardware-cycle sampling +and uses the same sharding mechanism. + +On an Arm64 host that advertises SVE, perf's automatic DWARF register set also +contains the extended `vg` pseudo-register. Linux's software clock PMUs reject +that extended register even though the hardware PMU accepts it. The default +`--perf-user-regs auto` therefore supplies every base Arm64 GPR while omitting +`vg` for `cpu-clock`/`task-clock`; explicit register lists are passed through, +and `perf-default` restores perf's own selection. Preflight uses a short real +`perf record`, not `perf stat`, so unsupported sampling/register combinations +fail before the benchmark begins. + +Artifact publication waits for completion by default and is ultimately bounded +by the Slurm job time limit, because prematurely killing a copy can lose the +only node-local raw capture. `--perf-finalize-timeout ` supplies an +explicit nonzero bound when desired. + +For copy-back automation, resolve the pointer once: + +```bash +run_dir="$(cat latest_result_dir.txt)" +rsync -aP "${run_dir}/" /destination/"${run_dir}/" + +# For a quick diagnostic copy that intentionally omits large perf artifacts: +rsync -aP --exclude '/profiles/perf/' "${run_dir}/" /destination/"${run_dir}/" +``` + +By default, `--perf` also fails preflight when the worker executable contains +neither `.symtab` nor `.debug_info`, since an address-only flamegraph is not a +useful result. Set `PERF_REQUIRE_SYMBOLS=0` only when intentionally collecting +raw addresses for later symbolization with a matching external symbol build. + --- ## Entry points (reference) @@ -227,6 +485,10 @@ presto-nvl72/ ├── functions.sh # Coordinator/worker/queries helpers ├── echo_helpers.sh # Logging helpers ├── profiler_functions.sh # nsys profiling helpers (bind-mounted into workers) +├── perf_profiler_functions.sh # query-side perf token handshake +├── perf-worker-sampler.sh # host-side worker-0 capture orchestration +├── perf-record-sharded.sh # bounded TID-shard perf record processes +├── process-perf-captures.sh # deferred perf report/flamegraph generation │ ├── # Slurm wrappers + inner execution scripts ├── gen-tpch-data.slurm # Step 1 sbatch entry @@ -237,7 +499,7 @@ presto-nvl72/ │ ├── # Runtime output ├── logs/ # Coord/worker/cli logs (created per run) -└── result_dir/ # Benchmark results (created per run) +└── result_dir_/ # Immutable benchmark results per Slurm job ``` --- diff --git a/presto/slurm/presto-nvl72/collect-coordinator-queries.sh b/presto/slurm/presto-nvl72/collect-coordinator-queries.sh new file mode 100755 index 00000000..f25a244b --- /dev/null +++ b/presto/slurm/presto-nvl72/collect-coordinator-queries.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Collect retained Presto query JSON from a live Slurm benchmark coordinator. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}" + +source ./defaults.env + +PORT="${PORT:-${CLUSTER_DEFAULT_PORT:-9200}}" +OUT_DIR="${OUT_DIR:-}" +JOB_ID="" +SERVER="" + +usage() { + cat < [-o output_dir] [-p port] + $0 --server http://host:port [-o output_dir] + +Fetches the coordinator's retained /v1/query list and one full +/v1/query/ JSON file per query into output_dir. + +Defaults: + output_dir: result_dir_/coordinator_queries (job ID mode) + coordinator_queries (--server mode) + port: ${PORT} +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -o|--output-dir) + [[ -n "${2:-}" ]] || { echo "Error: $1 requires a value" >&2; exit 2; } + OUT_DIR="$2" + shift 2 + ;; + -p|--port) + [[ -n "${2:-}" ]] || { echo "Error: $1 requires a value" >&2; exit 2; } + PORT="$2" + shift 2 + ;; + --server) + [[ -n "${2:-}" ]] || { echo "Error: $1 requires a value" >&2; exit 2; } + SERVER="${2%/}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + -*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + if [[ -z "${JOB_ID}" ]]; then + JOB_ID="$1" + else + echo "Unexpected argument: $1" >&2 + usage >&2 + exit 2 + fi + shift + ;; + esac +done + +if [[ -z "${JOB_ID}" && -z "${SERVER}" ]]; then + usage >&2 + exit 2 +fi + +# Default output directory incorporates the job ID when available so each run +# lands in the same result_dir_/ tree as the benchmark output. +if [[ -z "${OUT_DIR}" ]]; then + if [[ -n "${JOB_ID}" ]]; then + OUT_DIR="result_dir_${JOB_ID}/coordinator_queries" + else + OUT_DIR="coordinator_queries" + fi +fi + +if [[ -n "${JOB_ID}" ]]; then + NODELIST="$(squeue -j "${JOB_ID}" -h -o '%N' | awk 'NR==1{print $1}')" + if [[ -z "${NODELIST}" || "${NODELIST}" == "(null)" ]]; then + echo "Could not find allocated nodes for job ${JOB_ID}. Is it still running?" >&2 + exit 1 + fi + + COORD="$(scontrol show hostnames "${NODELIST}" | head -1)" + if [[ -z "${COORD}" ]]; then + echo "Could not resolve coordinator node from node list: ${NODELIST}" >&2 + exit 1 + fi + + SERVER="http://${COORD}:${PORT}" +fi + +mkdir -p "${OUT_DIR}/queries" + +[[ -n "${JOB_ID}" ]] && echo "Job: ${JOB_ID}" +[[ -n "${COORD:-}" ]] && echo "Coordinator: ${COORD}" +echo "Server: ${SERVER}" +echo "Output: ${OUT_DIR}" + +fetch_from_coord() { + local path="$1" + if [[ -n "${JOB_ID}" ]]; then + srun --jobid "${JOB_ID}" -w "${COORD}" --ntasks=1 --overlap \ + bash -lc "curl -fsS --max-time 30 '${SERVER}${path}'" + else + curl -fsS --max-time 30 "${SERVER}${path}" + fi +} + +echo "Fetching coordinator query list..." +fetch_from_coord "/v1/query" > "${OUT_DIR}/queries.json" +fetch_from_coord "/v1/info" > "${OUT_DIR}/info.json" || true +fetch_from_coord "/v1/node" > "${OUT_DIR}/nodes.json" || true + +mapfile -t QUERY_IDS < <( + python3 -c ' +import json, sys +with open(sys.argv[1]) as f: + data = json.load(f) +for q in data: + qid = q.get("queryId") + if qid: + print(qid) +' "${OUT_DIR}/queries.json" +) + +if [[ "${#QUERY_IDS[@]}" -eq 0 ]]; then + echo "No retained queries found in ${OUT_DIR}/queries.json" + exit 0 +fi + +echo "Fetching ${#QUERY_IDS[@]} full query JSON file(s)..." +for query_id in "${QUERY_IDS[@]}"; do + if fetch_from_coord "/v1/query/${query_id}" > "${OUT_DIR}/queries/${query_id}.json"; then + echo " ${query_id}" + else + echo " ${query_id} (failed to fetch)" >&2 + rm -f "${OUT_DIR}/queries/${query_id}.json" + fi +done + +python3 "${SCRIPT_DIR}/../../scripts/summarize_coordinator_queries.py" "${OUT_DIR}" + +echo "Wrote:" +echo " ${OUT_DIR}/queries.json" +echo " ${OUT_DIR}/queries/*.json" +echo " ${OUT_DIR}/operator_summaries/*.operators.json" +echo " ${OUT_DIR}/summary.tsv" diff --git a/presto/slurm/presto-nvl72/functions.sh b/presto/slurm/presto-nvl72/functions.sh index 86808ddd..07048a2b 100644 --- a/presto/slurm/presto-nvl72/functions.sh +++ b/presto/slurm/presto-nvl72/functions.sh @@ -2,6 +2,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +PRESTO_SCRIPT_FUNCTIONS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../scripts" && pwd)" +source "${PRESTO_SCRIPT_FUNCTIONS_DIR}/common_functions.sh" + +# Track only the srun clients started by this batch shell. Cleanup never uses a +# broad pkill, so it cannot terminate another user's process or an unrelated +# process in the allocation. +COORD_SRUN_PID="" +declare -a WORKER_SRUN_PIDS=() +declare -a WORKER_SRUN_IDS=() +declare -a WORKER_SRUN_NODES=() +declare -A CPU_NUMA_NODE_BY_WORKER_SLOT=() +declare -A CLUSTER_PORT_SPECS_BY_NODE=() +CLUSTER_STOPPED=0 + # Echo the ",:" bind-mount fragment for the host's # miniforge3 install if it exists, else nothing. Used by every container # that needs to run Python via init_python_virtual_env / run_py_script.sh: @@ -80,6 +94,38 @@ function generate_configs { echo "-XX:-UseContainerSupport" >> ${CONFIGS}/etc_common/jvm.config } +function resolve_node_ipv4_for_interface { + [ $# -ne 2 ] && echo_error "$0 expected arguments 'node' and 'interface'" + local node=$1 iface=$2 + srun -N1 -w "${node}" --ntasks=1 --overlap \ + /bin/bash -lc "ip -o -4 addr show dev '${iface}' | awk '{print \$4}' | cut -d/ -f1 | head -n 1" +} + +# Resolve an RDMA device (for example mlx5_0:1) to its Linux netdev and IPv4 +# address on a particular allocated node. This is deliberately performed on +# the compute node: NVL72 odd/even trays attach to different leaf groups, and +# interface names need not be globally encoded in the cluster configuration. +function resolve_node_ipv4_for_ucx_device { + [ $# -ne 2 ] && echo_error "$0 expected arguments 'node' and 'ucx_device'" + local node=$1 rdma_device=${2%%:*} + [[ "${rdma_device}" =~ ^[[:alnum:]_.-]+$ ]] || return 1 + + srun -N1 -w "${node}" --ntasks=1 --overlap \ + /bin/bash -lc ' + rdma_device=$1 + net_dir="/sys/class/infiniband/${rdma_device}/device/net" + [[ -d "${net_dir}" ]] || exit 1 + shopt -s nullglob + net_paths=("${net_dir}"/*) + ((${#net_paths[@]} > 0)) || exit 1 + iface="${net_paths[0]##*/}" + address="$(ip -o -4 addr show dev "${iface}" scope global \ + | sed -n "1{s/.* inet \\([^/ ]*\\).*/\\1/p;}")" + [[ -n "${address}" ]] || exit 1 + printf "%s %s\\n" "${iface}" "${address}" + ' _ "${rdma_device}" +} + # Takes a list of environment variables. Checks that each one is set and of non-zero length. function validate_environment_preconditions { local missing=() @@ -137,6 +183,8 @@ ${CONFIGS}/etc_coordinator/catalog/hive.properties:/opt/presto-server/etc/catalo ${DATA}:/var/lib/presto/data/hive/data/user_data,\ ${VT_ROOT}/.hive_metastore:/var/lib/presto/data/hive/metastore${extra_mounts} \ -- bash -lc "unset JAVA_HOME; export JAVA_HOME=/usr/lib/jvm/jre-17-openjdk; export PATH=/usr/lib/jvm/jre-17-openjdk/bin:\$PATH; ${script}" >> ${LOGS}/${log_file} 2>&1 & + COORD_SRUN_PID=$! + echo "Coordinator srun client PID: ${COORD_SRUN_PID}" else srun -w $COORD --ntasks=1 --overlap \ --container-remap-root \ @@ -206,17 +254,38 @@ run_coord_image "$COORD_SCRIPT" "coord" function run_worker { : "${ENABLE_GDS:=0}" : "${ENABLE_NSYS:=0}" - : "${NSYS_WORKER_ID:=0}" + : "${ENABLE_PERF:=0}" + : "${PERF_WORKER_ID:=0}" + # Migrate legacy NSYS_WORKER_ID (single int) → NSYS_WORKER_IDS (csv). + : "${NSYS_WORKER_IDS:=${NSYS_WORKER_ID:-0}}" - [ $# -ne 4 ] && echo_error "$0 expected arguments 'gpu_id', 'image', 'node_id', and 'worker_id'" + [ $# -ne 4 ] && echo_error "$0 expected arguments 'local_worker_id', 'image', 'node_id', and 'worker_id'" validate_environment_preconditions LOGS CONFIGS VT_ROOT COORD CUDF_LIB DATA - local gpu_id=$1 image=$2 node=$3 worker_id=$4 - # Pair each worker with the CPU NUMA domain its GPU is attached to. - # CLUSTER_NUMA_GPUS_PER_NODE controls how many GPUs share each NUMA domain - # (e.g., 2 means GPUs 0-1 on NUMA 0, GPUs 2-3 on NUMA 1). - local numa_node=$((gpu_id / ${CLUSTER_NUMA_GPUS_PER_NODE:-1})) - echo "running worker ${worker_id} with image ${image} on node ${node} with gpu_id ${gpu_id} numa_node ${numa_node}" + # The first argument is the worker's local slot on its host. For GPU jobs + # that slot is also the CUDA device id; for CPU jobs it is only an index + # into node-local interface/device lists. It restarts at zero on each host. + local local_worker_id=$1 image=$2 node=$3 worker_id=$4 + local gpu_id="${local_worker_id}" + local numa_node="" + if [[ "${VARIANT_TYPE}" == "gpu" ]]; then + # Pair each worker with the CPU NUMA domain its GPU is attached to. + # On GB200, GPUs 0-1 have CPU affinity 0-71 (node 0) and GPUs 2-3 + # have CPU affinity 72-143 (node 1). The GPU NUMA IDs reported by + # nvidia-smi are HBM nodes and are intentionally not used here. + numa_node=$((gpu_id / ${CLUSTER_NUMA_GPUS_PER_NODE:-1})) + elif [[ "${USE_NUMA}" == "1" ]]; then + numa_node="${CPU_NUMA_NODE_BY_WORKER_SLOT["${node}:${local_worker_id}"]:-}" + [[ -n "${numa_node}" ]] || \ + echo_error "Worker ${worker_id}: CPU NUMA placement was not prepared for ${node} local slot ${local_worker_id}" + fi + if [[ "${VARIANT_TYPE}" == "gpu" ]]; then + echo "running worker ${worker_id} with image ${image} on node ${node} with gpu_id ${gpu_id} numa_node ${numa_node}" + elif [[ "${USE_NUMA}" == "1" ]]; then + echo "running worker ${worker_id} with image ${image} on node ${node} with local_slot ${local_worker_id} cpu_numa_node ${numa_node}" + else + echo "running worker ${worker_id} with image ${image} on node ${node} with local_slot ${local_worker_id} (NUMA unbound)" + fi local worker_image worker_image=$(resolve_image_path "${image}") @@ -228,9 +297,93 @@ function run_worker { local worker_hive="${CONFIGS}/etc_worker_${worker_id}/catalog/hive.properties" local worker_data="${SCRIPT_DIR}/worker_data_${worker_id}" + # CPU UCX peers derive the destination listener port from the advertised + # Prestissimo HTTP task URL as HTTP+3. Derive the local listener from the + # same generated config rather than putting a single static port in the + # shared worker.env file; this stays correct for every worker ID. + local cpu_ucx_port="" + if [[ "${VARIANT_TYPE}" == "cpu" ]]; then + local worker_http_port + worker_http_port="$(awk -F= ' + /^[[:space:]]*http-server\.http\.port[[:space:]]*=/ { + gsub(/[[:space:]]/, "", $2) + print $2 + exit + } + ' "${worker_config}")" + if [[ ! "${worker_http_port}" =~ ^[0-9]+$ ]] || (( worker_http_port < 1 || worker_http_port > 65532 )); then + echo_error "Worker ${worker_id}: invalid or missing http-server.http.port in ${worker_config}" + fi + cpu_ucx_port=$((worker_http_port + 3)) + echo " Worker ${worker_id} CPU UCX listener: HTTP ${worker_http_port} + 3 = ${cpu_ucx_port}" + fi + # Each worker needs to be told how to access the coordianator sed -i "s+discovery\.uri.*+discovery\.uri=http://${COORD}:${PORT}+g" ${worker_config} + # Resolve a worker-specific UCX device from its local slot before entering + # the container. The same list is reused independently on every allocated + # node, so scheduling remains node-agnostic. Preserve the legacy per-GPU + # list and the generic UCX_NET_DEVICES multi-rail value as fallbacks. + local worker_ucx_net_device="${UCX_NET_DEVICES:-${CLUSTER_UCX_NET_DEVICES:-}}" + local ucx_net_devices_by_local_worker="${CLUSTER_UCX_NET_DEVICES_BY_LOCAL_WORKER:-}" + local ucx_net_devices_by_gpu="${CLUSTER_UCX_NET_DEVICES_BY_GPU:-}" + if [[ -n "${ucx_net_devices_by_local_worker}" ]]; then + IFS=',' read -ra ucx_net_device_entries <<< "${ucx_net_devices_by_local_worker}" + worker_ucx_net_device="${ucx_net_device_entries[${local_worker_id}]:-}" + [[ -n "${worker_ucx_net_device}" ]] || \ + echo_error "CLUSTER_UCX_NET_DEVICES_BY_LOCAL_WORKER has no entry for local worker ${local_worker_id}" + elif [[ -n "${ucx_net_devices_by_gpu}" ]]; then + IFS=',' read -ra ucx_net_device_entries <<< "${ucx_net_devices_by_gpu}" + worker_ucx_net_device="${ucx_net_device_entries[${gpu_id}]:-}" + [[ -n "${worker_ucx_net_device}" ]] || \ + echo_error "CLUSTER_UCX_NET_DEVICES_BY_GPU has no entry for GPU ${gpu_id}" + fi + [[ -n "${worker_ucx_net_device}" ]] && \ + echo " Worker ${worker_id} UCX device: ${worker_ucx_net_device}" + + # Keep an explicit interface override when supplied. Otherwise CPU UCX + # derives the advertised address from the first selected RDMA device on + # each actual host. For a multi-rail list this first device is also the + # listener/bootstrap rail; UCX remains free to use the full device list for + # data movement. The legacy per-GPU interface list remains available to GPU + # runs and to callers that intentionally want fixed local-slot pairing. + local internal_address_interface="${PRESTO_INTERNAL_ADDRESS_INTERFACE:-${CLUSTER_INTERNAL_ADDRESS_INTERFACE:-}}" + local internal_address_interface_by_gpu="${PRESTO_INTERNAL_ADDRESS_INTERFACE_BY_GPU:-${CLUSTER_INTERNAL_ADDRESS_INTERFACE_BY_GPU:-}}" + if [[ -n "${internal_address_interface_by_gpu}" ]]; then + local -a internal_address_interfaces + IFS=',' read -ra internal_address_interfaces <<< "${internal_address_interface_by_gpu}" + internal_address_interface="${internal_address_interfaces[${gpu_id}]:-}" + [[ -n "${internal_address_interface}" ]] || \ + echo_error "PRESTO_INTERNAL_ADDRESS_INTERFACE_BY_GPU/CLUSTER_INTERNAL_ADDRESS_INTERFACE_BY_GPU has no entry for GPU ${gpu_id}" + fi + + local internal_address="" address_source="" + if [[ -n "${internal_address_interface}" ]]; then + internal_address="$(resolve_node_ipv4_for_interface "${node}" "${internal_address_interface}")" + [[ -n "${internal_address}" ]] || \ + echo_error "Could not resolve ${internal_address_interface} IPv4 address on ${node}" + address_source="interface ${internal_address_interface}" + elif [[ "${VARIANT_TYPE}" == "cpu" && "${worker_ucx_net_device}" == mlx5_* ]]; then + local bootstrap_ucx_device="${worker_ucx_net_device%%,*}" + local discovered_address + discovered_address="$(resolve_node_ipv4_for_ucx_device "${node}" "${bootstrap_ucx_device}")" || \ + echo_error "Worker ${worker_id}: could not map ${bootstrap_ucx_device} to an IPv4 netdev on ${node}" + read -r internal_address_interface internal_address <<< "${discovered_address}" + [[ -n "${internal_address_interface}" && -n "${internal_address}" ]] || \ + echo_error "Worker ${worker_id}: incomplete RDMA address discovery for ${bootstrap_ucx_device} on ${node}" + address_source="auto ${bootstrap_ucx_device} -> ${internal_address_interface}" + fi + + if [[ -n "${internal_address}" ]]; then + if grep -q "^node\.internal-address=" "${worker_node}"; then + sed -i "s+^node\.internal-address=.*+node.internal-address=${internal_address}+g" "${worker_node}" + else + echo "node.internal-address=${internal_address}" >> "${worker_node}" + fi + echo " Worker ${worker_id} internal address: ${internal_address} (${address_source})" + fi + # Create unique data dir per worker. mkdir -p ${worker_data} mkdir -p ${worker_data}/hive/data/user_data @@ -239,6 +392,14 @@ function run_worker { local vt_worker_env_file="/var/worker_env_file" local vt_cufile_log_dir="/var/log/cufile" local vt_cufile_log="${vt_cufile_log_dir}/cufile_worker_${worker_id}.log" + local container_script_dir="${SCRIPT_DIR/${VT_ROOT}//workspace}" + [[ "${container_script_dir}" == /workspace/* ]] || \ + echo_error "SCRIPT_DIR must be below VT_ROOT (${VT_ROOT}) so the NUMA launcher is visible in the worker container: ${SCRIPT_DIR}" + local vt_numa_launcher="${container_script_dir}/numa-worker-launch.sh" + if [[ "${USE_NUMA:-0}" == "1" ]]; then + [[ -f "${SCRIPT_DIR}/numa-worker-launch.sh" ]] || \ + echo_error "NUMA worker launcher not found: ${SCRIPT_DIR}/numa-worker-launch.sh" + fi # GDS (GPU Direct Storage) is a GPU-only feature. On CPU variant, skip # the cufile/nvidia-fs bind-mounts even when ENABLE_GDS=1 — /etc/cufile.json @@ -257,12 +418,19 @@ function run_worker { local nsys_bin="" local nsys_launch_opts="" local vt_nsys_report_dir="/var/log/nsys" - if [[ "${ENABLE_NSYS}" == "1" && "${worker_id}" == "${NSYS_WORKER_ID}" ]]; then + local publish_perf_pid=0 + if [[ "${ENABLE_NSYS}" == "1" && ",${NSYS_WORKER_IDS}," == *",${worker_id},"* ]]; then # nsys must be on PATH inside the worker container. The recommended approach is # a symlink in the image build: # ln -sf /opt/nvidia/nsight-systems-cli//bin/nsys /usr/local/bin/nsys nsys_bin="nsys" - nsys_launch_opts="-t nvtx,cuda" + # nsys then uses its own built-in default trace set. + nsys_launch_opts="${NSYS_LAUNCH_OPTS:-}" + fi + if [[ "${ENABLE_PERF}" == "1" && "${worker_id}" == "${PERF_WORKER_ID}" ]]; then + publish_perf_pid=1 + # Remove a PID from a prior failed job before the new container starts. + rm -f "${LOGS}/.presto_worker_pid_w${worker_id}" fi # To re-enable verbose GLOG logging, add these flags to the srun call below @@ -297,13 +465,26 @@ function run_worker { fi # NVIDIA_VISIBLE_DEVICES triggers enroot's 98-nvidia.sh hook, which requires - # nvidia-container-cli on the host. CPU-only nodes typically don't have it, - # so the hook fails container start. Skip the export on the CPU variant. + # nvidia-container-cli on the host. CPU-only nodes typically do not have it, + # so this export remains GPU-only. local nvidia_env="" if [[ "${VARIANT_TYPE}" == "gpu" ]]; then nvidia_env=",NVIDIA_VISIBLE_DEVICES=all,NVIDIA_DRIVER_CAPABILITIES=compute,utility" fi + # Enroot's Mellanox hook exposes /dev/infiniband and the matching providers. + # GPU jobs default to the historical "all" behavior. Selecting an mlx5 + # device is also an explicit CPU opt-in: expose the matching RDMA devices + # and providers without requiring a second, redundant topology setting. + # Ordinary CPU-only hosts using UCX auto/TCP still avoid the hook. + if [[ -n "${CLUSTER_MELLANOX_VISIBLE_DEVICES:-}" ]]; then + export MELLANOX_VISIBLE_DEVICES="${CLUSTER_MELLANOX_VISIBLE_DEVICES}" + elif [[ "${VARIANT_TYPE}" == "gpu" || "${worker_ucx_net_device}" == *mlx5_* ]]; then + export MELLANOX_VISIBLE_DEVICES=all + else + unset MELLANOX_VISIBLE_DEVICES + fi + # Notes on nsys profiling # # In the worker container here, we use `nsys launch` to start the presto_server, @@ -333,6 +514,11 @@ function run_worker { # The token files live in /var/log/nsys, the same host directory bind-mounted # into both the worker and cli containers. + # Pyxis/Enroot share the host network and IPC namespaces unless explicitly + # asked to unshare them (and do not create a separate PID namespace here). + # That already matches Compose's network/ipc/pid host settings required by + # UCX same-host transports; do not add --container-unshare=ipc to this step. + UCX_NET_DEVICES="${worker_ucx_net_device}" \ srun -N1 -w $node --ntasks=1 --overlap \ --container-image=${worker_image} \ --container-remap-root \ @@ -349,12 +535,31 @@ ${WORKER_ENV_FILE}:${vt_worker_env_file},\ ${LOGS}:${vt_cufile_log_dir},\ ${LOGS}:${vt_nsys_report_dir}${driver_mounts}${gds_mounts:+,${gds_mounts}}${worker_extra_mounts} \ -- /bin/bash -c " -export LD_LIBRARY_PATH='${CUDF_LIB}':/usr/local/lib:\${LD_LIBRARY_PATH:-} - set -a source ${vt_worker_env_file} set +a +if [[ \"\${PRESTO_WORKER_UCX_LOCAL_LIB_FIRST:-0}\" == \"1\" ]]; then + export LD_LIBRARY_PATH=/usr/local/lib:'${CUDF_LIB}':\${LD_LIBRARY_PATH:-} +else + export LD_LIBRARY_PATH='${CUDF_LIB}':/usr/local/lib:\${LD_LIBRARY_PATH:-} +fi + +if [[ '${VARIANT_TYPE}' == 'cpu' ]]; then + if [[ -n \"\${VELOX_UCX_CPU_PORT:-}\" && \"\${VELOX_UCX_CPU_PORT}\" != '${cpu_ucx_port}' ]]; then + echo \"Worker ${worker_id}: overriding VELOX_UCX_CPU_PORT=\${VELOX_UCX_CPU_PORT} with generated HTTP+3 port ${cpu_ucx_port}\" + fi + export VELOX_UCX_CPU_EXCHANGE=\"\${VELOX_UCX_CPU_EXCHANGE:-1}\" + export VELOX_UCX_CPU_PORT='${cpu_ucx_port}' + if [[ \"\${VERIFY_CPU_UCX:-0}\" == '1' && \"\${VELOX_UCX_CPU_EXCHANGE}\" != '1' ]]; then + echo \"Worker ${worker_id}: --verify-cpu-ucx requires VELOX_UCX_CPU_EXCHANGE=1\" >&2 + exit 2 + fi + echo \"Worker ${worker_id}: CPU UCX enabled=\${VELOX_UCX_CPU_EXCHANGE}, port=\${VELOX_UCX_CPU_PORT}\" + echo \"Worker ${worker_id}: UCX_TLS=\${UCX_TLS:-unset}, UCX_NET_DEVICES=\${UCX_NET_DEVICES:-unset}\" + echo \"Worker ${worker_id}: UCX_MODULE_DIR=\${UCX_MODULE_DIR:-unset}\" +fi + if [[ '${VARIANT_TYPE}' == 'gpu' ]]; then export CUDA_VISIBLE_DEVICES=${gpu_id}; fi if [[ '${ENABLE_GDS}' == '1' ]]; then @@ -366,13 +571,29 @@ else fi echo \"Worker ${worker_id}: CUDA_VISIBLE_DEVICES=\${CUDA_VISIBLE_DEVICES:-none}, NUMA_NODE=${numa_node}\" +echo \"Worker ${worker_id}: UCX_NET_DEVICES=\${UCX_NET_DEVICES:-unset}\" echo \"Worker ${worker_id}: ENABLE_GDS=\${ENABLE_GDS:-unset}\" echo \"Worker ${worker_id}: ENABLE_NSYS=\${ENABLE_NSYS:-unset}\" +echo \"Worker ${worker_id}: ENABLE_PERF=\${ENABLE_PERF:-unset}\" echo \"Worker ${worker_id}: KVIKIO_COMPAT_MODE=\${KVIKIO_COMPAT_MODE:-unset}\" echo \"Worker ${worker_id}: CUFILE_LOGFILE_PATH=\${CUFILE_LOGFILE_PATH:-unset}\" echo \"Worker ${worker_id}: KVIKIO_TASK_SIZE=\${KVIKIO_TASK_SIZE:-unset}\" echo \"Worker ${worker_id}: KVIKIO_NTHREADS=\${KVIKIO_NTHREADS:-unset}\" +if [[ '${VARIANT_TYPE}' == 'cpu' && \"\${VERIFY_CPU_UCX:-0}\" == '1' ]]; then + echo \"Worker ${worker_id}: LD_LIBRARY_PATH=\${LD_LIBRARY_PATH:-unset}\" + if command -v ucx_info >/dev/null 2>&1; then + echo \"Worker ${worker_id}: UCX runtime version\" + ucx_info -v + else + echo \"Worker ${worker_id}: ucx_info not found\" >&2 + fi + if command -v ldd >/dev/null 2>&1; then + echo \"Worker ${worker_id}: presto_server UCX linkage\" + ldd /usr/bin/presto_server | grep -E 'libuc[mpxt]|not found' || true + fi +fi + if [[ -n '${nsys_bin}' ]]; then ( echo \"Worker ${worker_id}: nsys subshell started\" @@ -407,45 +628,669 @@ if [[ -n '${nsys_bin}' ]]; then echo \"Worker ${worker_id}: Nsight System program at ${nsys_bin}\" echo \"Worker ${worker_id}: running nsys launch\" if [[ '${USE_NUMA}' == '1' ]]; then - ${nsys_bin} launch ${nsys_launch_opts} numactl --cpubind=${numa_node} --membind=${numa_node} /usr/bin/presto_server --etc-dir=/opt/presto-server/etc + ${nsys_bin} launch ${nsys_launch_opts} /bin/bash ${vt_numa_launcher} ${numa_node} /usr/bin/presto_server --etc-dir=/opt/presto-server/etc else ${nsys_bin} launch ${nsys_launch_opts} /usr/bin/presto_server --etc-dir=/opt/presto-server/etc fi echo \"Worker ${worker_id}: nsys launch exited with code: \$?\" else - if [[ '${USE_NUMA}' == '1' ]]; then - numactl --cpubind=${numa_node} --membind=${numa_node} /usr/bin/presto_server --etc-dir=/opt/presto-server/etc + if [[ '${publish_perf_pid}' == '1' ]]; then + perf_pid_file='${vt_nsys_report_dir}/.presto_worker_pid_w${worker_id}' + rm -f \"\${perf_pid_file}\" + if [[ '${USE_NUMA}' == '1' ]]; then + /bin/bash ${vt_numa_launcher} ${numa_node} /usr/bin/presto_server --etc-dir=/opt/presto-server/etc & + else + /usr/bin/presto_server --etc-dir=/opt/presto-server/etc & + fi + server_pid=\$! + printf '%s\n' \"\${server_pid}\" > \"\${perf_pid_file}.tmp\" + mv \"\${perf_pid_file}.tmp\" \"\${perf_pid_file}\" + echo \"Worker ${worker_id}: published presto_server host PID \${server_pid} for perf\" + wait \"\${server_pid}\" + server_rc=\$? + rm -f \"\${perf_pid_file}\" + exit \"\${server_rc}\" + elif [[ '${USE_NUMA}' == '1' ]]; then + /bin/bash ${vt_numa_launcher} ${numa_node} /usr/bin/presto_server --etc-dir=/opt/presto-server/etc else /usr/bin/presto_server --etc-dir=/opt/presto-server/etc fi fi " > ${LOGS}/worker_${worker_id}.log 2>&1 & + WORKER_SRUN_PIDS+=("$!") + WORKER_SRUN_IDS+=("${worker_id}") + WORKER_SRUN_NODES+=("${node}") + echo " Worker ${worker_id} srun client PID: ${WORKER_SRUN_PIDS[$((${#WORKER_SRUN_PIDS[@]} - 1))]}" +} + +# ---------------------------------------------------------------------------- +# Host perf sidecar +# ---------------------------------------------------------------------------- + +# PID of the background srun that hosts perf-worker-sampler.sh. This is kept in +# the batch shell so success and failure paths can both finalize captures before +# Slurm tears down the allocation. +PERF_SIDECAR_SRUN_PID="" + +function start_perf_sampler { + [[ "${ENABLE_PERF:-0}" == "1" ]] || return 0 + + local worker_id="${PERF_WORKER_ID:-0}" + local sampler="${SCRIPT_DIR}/perf-worker-sampler.sh" + local sharded_recorder="${SCRIPT_DIR}/perf-record-sharded.sh" + local postprocess_script="${SCRIPT_DIR}/process-perf-captures.sh" + local result_dir="${RESULT_DIR:-${SCRIPT_DIR}/result_dir}" + local output_dir="${result_dir}/profiles/perf" + local ready_file="${LOGS}/.perf_sidecar_ready_w${worker_id}" + local error_file="${LOGS}/.perf_sidecar_error_w${worker_id}" + local done_file="${LOGS}/.perf_sidecar_done_w${worker_id}" + local sampler_log="${LOGS}/perf_worker_${worker_id}.log" + + [[ "${worker_id}" == "0" ]] || echo_error "This perf path currently supports worker 0 only" + [[ -f "${sampler}" ]] || echo_error "Perf sampler not found: ${sampler}" + [[ -f "${sharded_recorder}" ]] || echo_error "Sharded perf recorder not found: ${sharded_recorder}" + [[ -f "${postprocess_script}" ]] || echo_error "Perf postprocessor not found: ${postprocess_script}" + + mkdir -p "${output_dir}" + rm -f \ + "${ready_file}" \ + "${error_file}" \ + "${done_file}" \ + "${LOGS}/.perf_sidecar_shutdown_w${worker_id}" \ + "${LOGS}"/.perf_start_w"${worker_id}"_* \ + "${LOGS}"/.perf_started_w"${worker_id}"_* \ + "${LOGS}"/.perf_stop_w"${worker_id}"_* \ + "${LOGS}"/.perf_finished_w"${worker_id}"_* \ + "${LOGS}"/.perf_error_w"${worker_id}"_* + + echo "Starting host perf sidecar for worker ${worker_id} on ${COORD}..." + srun -N1 -w "${COORD}" --ntasks=1 --overlap --export=ALL \ + bash "${sampler}" \ + --worker-id "${worker_id}" \ + --control-dir "${LOGS}" \ + --output-dir "${output_dir}" \ + --postprocess-script "${postprocess_script}" \ + --postprocess "${PERF_POSTPROCESS:-0}" \ + --event "${PERF_EVENT:-cpu-clock:u}" \ + --frequency "${PERF_FREQUENCY:-19}" \ + --call-graph "${PERF_CALL_GRAPH:-dwarf,8192}" \ + --user-regs "${PERF_USER_REGS:-auto}" \ + --nofile-limit "${PERF_NOFILE_LIMIT:-65536}" \ + --threads-per-shard "${PERF_THREADS_PER_SHARD:-128}" \ + --record-stop-timeout "${PERF_RECORD_STOP_TIMEOUT:-120}" \ + >"${sampler_log}" 2>&1 & + PERF_SIDECAR_SRUN_PID=$! + + local waited + for waited in {1..180}; do + if [[ -f "${ready_file}" ]]; then + echo "Host perf sidecar is ready for worker ${worker_id}." + return 0 + fi + if [[ -s "${error_file}" ]]; then + cat "${error_file}" >&2 + echo_error "Host perf preflight failed; see ${sampler_log}" + fi + if ! kill -0 "${PERF_SIDECAR_SRUN_PID}" 2>/dev/null; then + wait "${PERF_SIDECAR_SRUN_PID}" || true + PERF_SIDECAR_SRUN_PID="" + echo_error "Host perf sidecar exited before becoming ready; see ${sampler_log}" + fi + sleep 1 + done + echo_error "Timed out waiting for host perf sidecar; see ${sampler_log}" +} + +function stop_perf_sampler { + [[ "${ENABLE_PERF:-0}" == "1" ]] || return 0 + + local worker_id="${PERF_WORKER_ID:-0}" + local shutdown_file="${LOGS}/.perf_sidecar_shutdown_w${worker_id}" + local done_file="${LOGS}/.perf_sidecar_done_w${worker_id}" + local error_file="${LOGS}/.perf_sidecar_error_w${worker_id}" + local sampler_log="${LOGS}/perf_worker_${worker_id}.log" + local finalize_timeout="${PERF_FINALIZE_TIMEOUT:-0}" + local sidecar_rc=0 + local timeout_rc=0 + + if [[ -z "${PERF_SIDECAR_SRUN_PID}" && -f "${done_file}" ]]; then + if [[ -s "${error_file}" ]]; then + cat "${error_file}" >&2 + return 1 + fi + return 0 + fi + [[ "${finalize_timeout}" =~ ^[0-9]+$ ]] || { + echo "Invalid PERF_FINALIZE_TIMEOUT=${finalize_timeout}; expected a non-negative integer" >&2 + return 1 + } + + touch "${shutdown_file}" + + if [[ -n "${PERF_SIDECAR_SRUN_PID}" ]]; then + if (( finalize_timeout > 0 )); then + if command -v timeout >/dev/null 2>&1; then + timeout --foreground "${finalize_timeout}s" bash -c ' + while kill -0 "$1" 2>/dev/null; do + sleep 1 + done + ' bash "${PERF_SIDECAR_SRUN_PID}" || timeout_rc=$? + else + local deadline=$((SECONDS + finalize_timeout)) + while kill -0 "${PERF_SIDECAR_SRUN_PID}" 2>/dev/null && (( SECONDS < deadline )); do + sleep 1 + done + if kill -0 "${PERF_SIDECAR_SRUN_PID}" 2>/dev/null; then + timeout_rc=124 + fi + fi + if (( timeout_rc == 124 )); then + echo "Timed out after ${finalize_timeout}s waiting for perf artifact publication; terminating its srun. See ${sampler_log}" >&2 + kill -TERM "${PERF_SIDECAR_SRUN_PID}" 2>/dev/null || true + elif (( timeout_rc != 0 )); then + echo "Error while waiting for perf artifact publication (status ${timeout_rc}); see ${sampler_log}" >&2 + kill -TERM "${PERF_SIDECAR_SRUN_PID}" 2>/dev/null || true + fi + fi + wait "${PERF_SIDECAR_SRUN_PID}" || sidecar_rc=$? + PERF_SIDECAR_SRUN_PID="" + fi + if [[ ! -f "${done_file}" ]]; then + echo "Perf sidecar did not publish its completion token; see ${sampler_log}" >&2 + return 1 + fi + if [[ -s "${error_file}" ]]; then + cat "${error_file}" >&2 + return 1 + fi + if (( sidecar_rc != 0 )); then + echo "Perf sidecar srun exited with status ${sidecar_rc}; see ${sampler_log}" >&2 + return "${sidecar_rc}" + fi + echo "Host perf raw captures published. Artifacts: ${RESULT_DIR:-${SCRIPT_DIR}/result_dir}/profiles/perf/worker_${worker_id}" } # ---------------------------------------------------------------------------- # Cluster lifecycle # ---------------------------------------------------------------------------- + +# Return success only while a tracked child exists and is not a zombie waiting +# for this shell to reap it. +function tracked_process_is_running { + local pid="${1:-}" state="" + [[ "${pid}" =~ ^[1-9][0-9]*$ ]] || return 1 + kill -0 "${pid}" 2>/dev/null || return 1 + state="$(ps -o stat= -p "${pid}" 2>/dev/null | tr -d '[:space:]')" + [[ -n "${state}" && "${state}" != Z* ]] +} + +# Discover the CPU-bearing NUMA nodes independently on every selected host. +# The generated worker configs are homogeneous, so reject a heterogeneous +# allocation instead of silently applying resource limits tuned for another +# node's topology. +function prepare_cpu_numa_layout { + CPU_NUMA_NODE_BY_WORKER_SLOT=() + [[ "${VARIANT_TYPE:-gpu}" == "cpu" ]] || return 0 + + if [[ "${USE_NUMA:-0}" != "1" ]]; then + echo "CPU NUMA binding is disabled; workers may use every CPU and allowed memory node on their host." + return 0 + fi + local common_functions="${PRESTO_SCRIPT_FUNCTIONS_DIR}/common_functions.sh" + local reference_signature="" node output line + for node in $(scontrol show hostnames "${SLURM_JOB_NODELIST}"); do + if ! output="$( + srun -N1 -w "${node}" --ntasks=1 --overlap \ + /bin/bash -s -- "${common_functions}" "${NUM_GPUS_PER_NODE}" <<'EOS' +set -euo pipefail +source "$1" +workers_per_host="$2" +discover_cpu_numa_topology + +cpu_ids="$(IFS=,; printf '%s' "${CPU_NUMA_NODE_IDS[*]}")" +cpu_counts="$(IFS=,; printf '%s' "${CPU_NUMA_NODE_CPU_COUNTS[*]}")" +cpu_memory="$(IFS=,; printf '%s' "${CPU_NUMA_NODE_MEMORY_GB[*]}")" +memory_ids="$(IFS=,; printf '%s' "${MEMORY_NUMA_NODE_IDS[*]}")" +printf 'TOPOLOGY|%s|%s|%s|%s|%s\n' \ + "${VISIBLE_NUMA_NODE_COUNT}" "${cpu_ids}" "${cpu_counts}" \ + "${cpu_memory}" "${memory_ids}" + +for ((slot = 0; slot < workers_per_host; ++slot)); do + printf 'MAP|%s|%s\n' \ + "${slot}" "$(cpu_numa_node_for_worker "${slot}" "${workers_per_host}")" +done +EOS + )"; then + echo "${output}" >&2 + echo_error "Could not discover CPU NUMA topology on ${node}" + fi + + local signature="" visible="" cpu_ids="" cpu_counts="" cpu_memory="" memory_ids="" + local slot="" cpu_node="" map_count=0 mapping="" + while IFS= read -r line; do + case "${line}" in + TOPOLOGY\|*) + IFS='|' read -r _ visible cpu_ids cpu_counts cpu_memory memory_ids <<< "${line}" + signature="${visible}|${cpu_ids}|${cpu_counts}|${cpu_memory}|${memory_ids}" + ;; + MAP\|*) + IFS='|' read -r _ slot cpu_node <<< "${line}" + if [[ ! "${slot}" =~ ^[0-9]+$ || ! "${cpu_node}" =~ ^[0-9]+$ ]]; then + echo_error "Invalid CPU NUMA mapping returned by ${node}: ${line}" + fi + CPU_NUMA_NODE_BY_WORKER_SLOT["${node}:${slot}"]="${cpu_node}" + mapping+="${mapping:+,}${slot}->${cpu_node}" + map_count=$((map_count + 1)) + ;; + "") + ;; + *) + echo "CPU NUMA discovery output from ${node}: ${line}" + ;; + esac + done <<< "${output}" + + [[ -n "${signature}" ]] || echo_error "CPU NUMA discovery on ${node} returned no topology record" + (( map_count == NUM_GPUS_PER_NODE )) || \ + echo_error "CPU NUMA discovery on ${node} returned ${map_count} worker mappings; expected ${NUM_GPUS_PER_NODE}" + + if [[ -z "${reference_signature}" ]]; then + reference_signature="${signature}" + elif [[ "${signature}" != "${reference_signature}" ]]; then + echo_error "CPU NUMA topology on ${node} differs from the coordinator host; heterogeneous CPU resource tuning is not supported" + fi + + echo "CPU NUMA topology on ${node}: visible=${visible}, CPU+DRAM nodes=${cpu_ids} (CPUs=${cpu_counts}, DRAM_GB=${cpu_memory}), all memory nodes=${memory_ids}" + echo "CPU NUMA worker mapping on ${node}: ${mapping}" + done +} + +# Inspect fixed coordinator, worker HTTP, and HTTP+3 exchange ports on one +# compute node. require-free waits until every port is reusable; require-bound +# waits until every selected port has been claimed. Exchange ports get an +# actual wildcard bind probe in addition to ss: UCX may own the sockaddr +# through a non-TCP listener, and a recently closed UCX listener can have no +# LISTEN entry while kernel socket state still makes a new bind fail with +# EADDRINUSE. The report includes Slurm cgroups, command lines, and matching +# TCP socket state where the kernel permits ownership inspection. +function run_port_inspection { + [ $# -ne 5 ] && echo_error "$0 expected arguments 'node', 'specs', 'mode', 'output_file', and 'wait_seconds'" + local node=$1 specs=$2 mode=$3 output_file=$4 wait_seconds=$5 + srun -N1 -w "${node}" --ntasks=1 --overlap \ + /bin/bash -s -- "${node}" "${specs}" "${mode}" "${wait_seconds}" \ + >"${output_file}" 2>&1 <<'EOS' +set -uo pipefail +node="$1" +specs="$2" +mode="$3" +wait_seconds="$4" + +case "${mode}" in + report|require-free|require-bound) + ;; + *) + echo "Port inspection on ${node}: invalid mode ${mode}" >&2 + exit 2 + ;; +esac +if ! command -v ss >/dev/null 2>&1; then + echo "Port inspection on ${node}: ss is not installed" >&2 + exit 2 +fi +if ! command -v python3 >/dev/null 2>&1; then + echo "Port inspection on ${node}: python3 is required for exchange-port bind probes" >&2 + exit 2 +fi + +probe_bind() { + python3 - "$1" <<'PY' +import socket +import sys + +port = int(sys.argv[1]) +try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("0.0.0.0", port)) +except OSError as error: + print( + f"bind(0.0.0.0:{port}) failed: " + f"[errno {error.errno}] {error.strerror}") + raise SystemExit(1) +PY +} + +inspect_once() { + local mismatch=0 entry role port matches pids pid bind_error connections state show_details + IFS=',' read -ra entries <<< "${specs}" + for entry in "${entries[@]}"; do + role="${entry%%:*}" + port="${entry##*:}" + state="free" + bind_error="" + connections="" + show_details=0 + matches="$(ss -H -ltnp 2>/dev/null \ + | awk -v wanted="${port}" '{ + count = split($4, fields, ":") + if (fields[count] == wanted) print + }')" + if [[ -n "${matches}" ]]; then + state="listen" + elif [[ "${role}" == *-exchange ]] && ! bind_error="$(probe_bind "${port}" 2>&1)"; then + state="bind-blocked" + connections="$(ss -H -tanp 2>/dev/null \ + | awk -v wanted="${port}" '{ + count = split($4, fields, ":") + if (fields[count] == wanted) print + }')" + fi + + case "${mode}" in + report) + case "${state}" in + listen) + echo "BUSY ${node} ${role} port ${port}" + show_details=1 + ;; + bind-blocked) + echo "BIND-BLOCKED ${node} ${role} port ${port}" + [[ -n "${bind_error}" ]] && echo "${bind_error}" + [[ -n "${connections}" ]] && echo "${connections}" + ;; + free) + echo "FREE ${node} ${role} port ${port}" + ;; + esac + ;; + require-free) + if [[ "${state}" == "listen" ]]; then + mismatch=1 + echo "BUSY ${node} ${role} port ${port}" + show_details=1 + elif [[ "${state}" == "bind-blocked" ]]; then + mismatch=1 + echo "BIND-BLOCKED ${node} ${role} port ${port}" + [[ -n "${bind_error}" ]] && echo "${bind_error}" + [[ -n "${connections}" ]] && echo "${connections}" + fi + ;; + require-bound) + if [[ "${state}" == "free" ]]; then + mismatch=1 + echo "MISSING ${node} ${role} port ${port}" + else + echo "READY ${node} ${role} port ${port} (${state})" + fi + ;; + esac + + if (( show_details == 1 )); then + echo "${matches}" + if command -v fuser >/dev/null 2>&1; then + pids="$(fuser -n tcp "${port}" 2>/dev/null || true)" + for pid in ${pids}; do + [[ "${pid}" =~ ^[0-9]+$ ]] || continue + ps -p "${pid}" -o pid=,ppid=,user=,stat=,lstart=,args= 2>/dev/null || true + if [[ -r "/proc/${pid}/cgroup" ]]; then + echo " cgroup:" + sed 's/^/ /' "/proc/${pid}/cgroup" + fi + if [[ -r "/proc/${pid}/cmdline" ]]; then + echo -n " cmdline: " + tr '\0' ' ' < "/proc/${pid}/cmdline" + echo + fi + done + fi + fi + done + return "${mismatch}" +} + +deadline=$((SECONDS + wait_seconds)) +while true; do + report="$(inspect_once)" + status=$? + if (( status == 0 )); then + [[ -n "${report}" ]] && echo "${report}" + exit 0 + fi + if [[ "${mode}" == "report" ]] || (( SECONDS >= deadline )); then + echo "${report}" + exit "${status}" + fi + sleep 1 +done +EOS +} + +function preflight_cluster_ports { + CLUSTER_PORT_SPECS_BY_NODE=() + local preflight_timeout="${PRESTO_PORT_PREFLIGHT_TIMEOUT:-${PRESTO_PORT_RELEASE_TIMEOUT:-90}}" + [[ "${preflight_timeout}" =~ ^[0-9]+$ ]] || \ + echo_error "Invalid PRESTO_PORT_PREFLIGHT_TIMEOUT=${preflight_timeout}" + + local worker_id=0 node local_worker_id config http_port exchange_port configured_exchange_port specs log_file + for node in $(scontrol show hostnames "${SLURM_JOB_NODELIST}"); do + specs="" + if [[ "${node}" == "${COORD}" ]]; then + specs="coordinator:${PORT}" + fi + for ((local_worker_id = 0; local_worker_id < NUM_GPUS_PER_NODE; ++local_worker_id)); do + config="${CONFIGS}/etc_worker_${worker_id}/config_native.properties" + http_port="$(awk -F= '/^http-server\.http\.port=/ {gsub(/[[:space:]]/, "", $2); print $2; exit}' "${config}")" + [[ "${http_port}" =~ ^[1-9][0-9]*$ ]] || \ + echo_error "Worker ${worker_id}: invalid HTTP port in ${config}" + (( http_port <= 65532 )) || \ + echo_error "Worker ${worker_id}: HTTP port ${http_port} leaves no valid HTTP+3 exchange port" + exchange_port=$((http_port + 3)) + configured_exchange_port="$(awk -F= '/^cudf\.exchange\.server\.port=/ {gsub(/[[:space:]]/, "", $2); print $2; exit}' "${config}")" + if [[ -n "${configured_exchange_port}" && "${configured_exchange_port}" != "${exchange_port}" ]]; then + echo_error "Worker ${worker_id}: configured exchange port ${configured_exchange_port} does not match HTTP+3 (${exchange_port})" + fi + specs+="${specs:+,}worker${worker_id}-http:${http_port},worker${worker_id}-exchange:${exchange_port}" + worker_id=$((worker_id + 1)) + done + CLUSTER_PORT_SPECS_BY_NODE["${node}"]="${specs}" + log_file="${LOGS}/port_preflight_${node}.log" + if ! run_port_inspection "${node}" "${specs}" require-free "${log_file}" "${preflight_timeout}"; then + cat "${log_file}" >&2 + echo_error "Fixed Presto/UCX ports are not bindable on ${node}; see ${log_file}" + fi + echo "Port preflight passed on ${node}: ${specs}" + done +} + +function collect_startup_diagnostics { + local reason="${1:-startup failure}" node specs log_file index + echo "Collecting cluster startup diagnostics: ${reason}" >&2 + echo "---- coord.log (last 120 lines) ----" >&2 + tail -n 120 "${LOGS}/coord.log" 2>/dev/null >&2 || true + for ((index = 0; index < ${#WORKER_SRUN_IDS[@]}; ++index)); do + echo "---- worker_${WORKER_SRUN_IDS[${index}]}.log on ${WORKER_SRUN_NODES[${index}]} (last 120 lines) ----" >&2 + tail -n 120 "${LOGS}/worker_${WORKER_SRUN_IDS[${index}]}.log" 2>/dev/null >&2 || true + done + for node in $(scontrol show hostnames "${SLURM_JOB_NODELIST}"); do + specs="${CLUSTER_PORT_SPECS_BY_NODE["${node}"]:-}" + [[ -n "${specs}" ]] || continue + log_file="${LOGS}/startup_ports_${node}.log" + run_port_inspection "${node}" "${specs}" report "${log_file}" 0 || true + echo "---- listener ownership on ${node} ----" >&2 + cat "${log_file}" >&2 || true + done +} + +# Stop exactly the coordinator and worker srun clients started above, wait for +# their steps to disappear, and verify that their fixed ports have been +# released before the batch allocation exits. +function stop_cluster { + [[ "${CLUSTER_STOPPED}" == "0" ]] || return 0 + CLUSTER_STOPPED=1 + + local stop_timeout="${PRESTO_CLUSTER_STOP_TIMEOUT:-30}" + local port_timeout="${PRESTO_PORT_RELEASE_TIMEOUT:-90}" + [[ "${stop_timeout}" =~ ^[0-9]+$ ]] || { + echo "Invalid PRESTO_CLUSTER_STOP_TIMEOUT=${stop_timeout}" >&2 + return 1 + } + [[ "${port_timeout}" =~ ^[0-9]+$ ]] || { + echo "Invalid PRESTO_PORT_RELEASE_TIMEOUT=${port_timeout}" >&2 + return 1 + } + + local -a tracked_pids=() + local pid index deadline any_running=0 cleanup_rc=0 node specs log_file + for ((index = ${#WORKER_SRUN_PIDS[@]} - 1; index >= 0; --index)); do + pid="${WORKER_SRUN_PIDS[${index}]}" + [[ -n "${pid}" ]] || continue + tracked_pids+=("${pid}") + if tracked_process_is_running "${pid}"; then + echo "Stopping worker ${WORKER_SRUN_IDS[${index}]} srun PID ${pid}..." + kill -TERM "${pid}" 2>/dev/null || true + fi + done + if [[ -n "${COORD_SRUN_PID}" ]]; then + tracked_pids+=("${COORD_SRUN_PID}") + if tracked_process_is_running "${COORD_SRUN_PID}"; then + echo "Stopping coordinator srun PID ${COORD_SRUN_PID}..." + kill -TERM "${COORD_SRUN_PID}" 2>/dev/null || true + fi + fi + + deadline=$((SECONDS + stop_timeout)) + while true; do + any_running=0 + for pid in "${tracked_pids[@]}"; do + if tracked_process_is_running "${pid}"; then + any_running=1 + break + fi + done + (( any_running == 0 || SECONDS >= deadline )) && break + sleep 1 + done + + if (( any_running != 0 )); then + echo "Cluster steps did not stop within ${stop_timeout}s; sending KILL to the tracked srun clients." >&2 + for pid in "${tracked_pids[@]}"; do + tracked_process_is_running "${pid}" && kill -KILL "${pid}" 2>/dev/null || true + done + cleanup_rc=1 + fi + for pid in "${tracked_pids[@]}"; do + wait "${pid}" 2>/dev/null || true + done + COORD_SRUN_PID="" + WORKER_SRUN_PIDS=() + + for node in $(scontrol show hostnames "${SLURM_JOB_NODELIST}"); do + specs="${CLUSTER_PORT_SPECS_BY_NODE["${node}"]:-}" + [[ -n "${specs}" ]] || continue + log_file="${LOGS}/port_cleanup_${node}.log" + if ! run_port_inspection "${node}" "${specs}" require-free "${log_file}" "${port_timeout}"; then + echo "Ports remained non-bindable after tracked-step cleanup on ${node}:" >&2 + cat "${log_file}" >&2 || true + cleanup_rc=1 + else + echo "Cluster ports released on ${node}." + fi + done + + return "${cleanup_rc}" +} + +# When requested by launch-run.sh --verify-cpu-ucx, require every expected +# HTTP+3 UCX port to be claimed before query submission. Do not infer readiness +# from UCX log strings: their presence depends on log level and UCX version, +# which made the old verifier reject healthy, fully registered clusters. +function verify_cpu_ucx_worker_startup { + [[ "${VERIFY_CPU_UCX:-0}" == "1" && "${VARIANT_TYPE:-gpu}" == "cpu" ]] || return 0 + + local ready_timeout="${PRESTO_CPU_UCX_READY_TIMEOUT:-60}" + [[ "${ready_timeout}" =~ ^[0-9]+$ ]] || \ + echo_error "Invalid PRESTO_CPU_UCX_READY_TIMEOUT=${ready_timeout}" + + local node specs exchange_specs entry role log_file + local -a entries + for node in $(scontrol show hostnames "${SLURM_JOB_NODELIST}"); do + specs="${CLUSTER_PORT_SPECS_BY_NODE["${node}"]:-}" + exchange_specs="" + IFS=',' read -ra entries <<< "${specs}" + for entry in "${entries[@]}"; do + role="${entry%%:*}" + [[ "${role}" == worker*-exchange ]] || continue + exchange_specs+="${exchange_specs:+,}${entry}" + done + [[ -n "${exchange_specs}" ]] || \ + echo_error "CPU UCX verification found no expected exchange ports on ${node}" + + log_file="${LOGS}/cpu_ucx_readiness_${node}.log" + if ! run_port_inspection "${node}" "${exchange_specs}" require-bound "${log_file}" "${ready_timeout}"; then + cat "${log_file}" >&2 || true + collect_startup_diagnostics "CPU UCX listener readiness failed on ${node}" + echo_error "CPU UCX listener readiness verification failed on ${node}; see ${log_file}" + fi + cat "${log_file}" + done + echo "CPU UCX listener ports verified on all ${NUM_WORKERS} worker(s)." +} + +# For multi-worker runs, require a retained coordinator query to report at +# least one CPU UCX replacement operator. Startup verification above proves +# every listener was ready before query submission; this check proves the +# selected query actually used the CPU-row UCX path. A one-worker plan cannot +# prove inter-worker transport and is therefore limited to listener readiness. +function verify_cpu_ucx_runtime { + [[ "${VERIFY_CPU_UCX:-0}" == "1" && "${VARIANT_TYPE:-gpu}" == "cpu" ]] || return 0 + + [[ "${NUM_WORKERS:-}" =~ ^[1-9][0-9]*$ ]] || \ + echo_error "CPU UCX verification requires a positive NUM_WORKERS value" + if (( NUM_WORKERS == 1 )); then + echo "CPU UCX runtime operator verification skipped for one worker; listener readiness was verified before query submission." + return 0 + fi + + local query_details_dir="${RESULT_DIR:-${SCRIPT_DIR}/result_dir}/coordinator_queries/queries" + if ! grep -R -E -q \ + '"operatorType"[[:space:]]*:[[:space:]]*"UcxCpuRow(Exchange|PartitionedOutput)"' \ + "${query_details_dir}" 2>/dev/null; then + echo "CPU UCX verification found no UcxCpuRowExchange or UcxCpuRowPartitionedOutput operator." >&2 + echo "Run an exchange-heavy query (for example TPC-H Q18) with at least two workers." >&2 + echo_error "CPU UCX operator verification failed" + fi + echo "CPU UCX replacement operator verified in retained query details." +} + # Start the coordinator, wait for it to come up, fan out NUM_GPUS_PER_NODE -# workers per node across $SLURM_JOB_NODELIST, then block until all +# local worker slots per node across $SLURM_JOB_NODELIST, then block until all # $NUM_WORKERS register. Used by both the benchmark and analyze flows. function start_cluster { + prepare_cpu_numa_layout + preflight_cluster_ports + echo "Starting Presto coordinator on ${COORD}..." run_coordinator wait_until_coordinator_is_running echo "Starting ${NUM_WORKERS} Presto workers across ${NUM_NODES} nodes..." - local worker_id=0 node gpu_id + local worker_id=0 node local_worker_id for node in $(scontrol show hostnames "$SLURM_JOB_NODELIST"); do - for gpu_id in $(seq 0 $((NUM_GPUS_PER_NODE - 1))); do - echo " Starting worker ${worker_id} on node ${node} GPU ${gpu_id}" - run_worker "${gpu_id}" "$WORKER_IMAGE" "${node}" "$worker_id" + for local_worker_id in $(seq 0 $((NUM_GPUS_PER_NODE - 1))); do + if [[ "${VARIANT_TYPE}" == "gpu" ]]; then + echo " Starting worker ${worker_id} on node ${node} GPU ${local_worker_id}" + else + echo " Starting worker ${worker_id} on node ${node} local slot ${local_worker_id}" + fi + run_worker "${local_worker_id}" "$WORKER_IMAGE" "${node}" "$worker_id" worker_id=$((worker_id + 1)) done done echo "Waiting for ${NUM_WORKERS} workers to register with coordinator..." wait_for_workers_to_register "$NUM_WORKERS" + verify_cpu_ucx_worker_startup + start_perf_sampler } # ---------------------------------------------------------------------------- @@ -532,65 +1377,79 @@ function run_queries { # (VT_ROOT is bind-mounted as /workspace) so the path stays correct even # if this directory is renamed away from `presto-nvl72`. local container_script_dir="${SCRIPT_DIR/${VT_ROOT}//workspace}" + local result_dir="${RESULT_DIR:-${SCRIPT_DIR}/result_dir}" + local container_result_dir="${result_dir/${VT_ROOT}//workspace}" + [[ "${container_result_dir}" == /workspace/* ]] || \ + echo_error "RESULT_DIR must be below VT_ROOT (${VT_ROOT}) so it is mounted in the CLI container: ${result_dir}" + + source "${SCRIPT_DIR}/defaults.env" local extra_args=() [[ "${ENABLE_METRICS}" == "1" ]] && extra_args+=("-m") - [[ "${ENABLE_NSYS}" == "1" ]] && extra_args+=("-p" "--profile-script-path" "${container_script_dir}/profiler_functions.sh") + if [[ "${ENABLE_NSYS}" == "1" ]]; then + extra_args+=("-p" "--profile-script-path" "${container_script_dir}/profiler_functions.sh") + elif [[ "${ENABLE_PERF:-0}" == "1" ]]; then + extra_args+=("-p" "--profile-script-path" "${container_script_dir}/perf_profiler_functions.sh") + fi [[ -n "${QUERIES:-}" ]] && extra_args+=("-q" "${QUERIES}") - - source "${SCRIPT_DIR}/defaults.env" - - # The upstream coordinator image ships without jq, which - # run_benchmark.sh's wait_for_worker_node_registration requires. - # yum/dnf cannot install it at runtime because the container root is - # a read-only squashfs (/var/cache/dnf is read-only). Stage a - # statically-linked jq under VT_ROOT (which is bind-mounted into the - # container as /workspace) and prepend that to PATH. The download - # is cached across runs so the cost is paid once. - local jq_cache="${VT_ROOT}/.cache/bin" - local jq_arch - case "$(uname -m)" in - aarch64|arm64) jq_arch="arm64" ;; - x86_64|amd64) jq_arch="amd64" ;; - *) echo_error "unsupported arch for jq download: $(uname -m)" ;; - esac - if [ ! -x "${jq_cache}/jq" ]; then - echo "Staging static jq (${jq_arch}) at ${jq_cache}/jq" - mkdir -p "${jq_cache}" - curl -sSL "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${jq_arch}" \ - -o "${jq_cache}/jq" - chmod +x "${jq_cache}/jq" + if [[ -n "${QUERIES_FILE:-}" ]]; then + local container_queries_file="${QUERIES_FILE}" + if [[ "${container_queries_file}" == "${VT_ROOT}/"* ]]; then + container_queries_file="/workspace/${container_queries_file#"${VT_ROOT}/"}" + fi + [[ "${container_queries_file}" == /workspace/* ]] || \ + echo_error "QUERIES_FILE must be below VT_ROOT (${VT_ROOT}) so it is mounted in the CLI container: ${QUERIES_FILE}" + extra_args+=("--queries-file" "${container_queries_file}") fi + local benchmark_args=( + ./run_benchmark.sh + -b tpch + -s "tpchsf${scale_factor}" + -i "${num_iterations}" + "${extra_args[@]}" + --hostname "${COORD}" + --port "${PORT}" + -o "${container_result_dir}" + --skip-drop-cache + ) + local benchmark_cmd + printf -v benchmark_cmd '%q ' "${benchmark_args[@]}" + # Result validation is intentionally not wired here yet: that belongs # to PR #275 (upstream validate_results.py) and will be hooked up # after the PR merges and this branch is rebased. # Cache-drop is skipped because it requires docker (not available on # the cluster). - run_coord_image "export PATH=/workspace/.cache/bin:\$PATH; \ - export PORT=$PORT; \ + run_coord_image "export PORT=$PORT; \ export HOSTNAME=$COORD; \ export PRESTO_DATA_DIR=/var/lib/presto/data/hive/data/user_data; \ export MINIFORGE_HOME=/workspace/miniforge3; \ export HOME=/workspace; \ cd /workspace/presto/scripts; \ - ./run_benchmark.sh -b tpch -s tpchsf${scale_factor} -i ${num_iterations} ${extra_args[*]} \ - --hostname ${COORD} --port $PORT -o ${container_script_dir}/result_dir --skip-drop-cache" "cli" + ${benchmark_cmd}" "cli" } # Check if the coordinator is running via curl. Fail after 10 retries. function wait_until_coordinator_is_running { echo "waiting for coordinator to be accessible" validate_environment_preconditions COORD LOGS - local state="INACTIVE" + local state="INACTIVE" coord_rc=0 for i in {1..10}; do state=$(curl -s http://${COORD}:${PORT}/v1/info/state || true) if [[ "$state" == "\"ACTIVE\"" ]]; then echo "coord started. state: $state" return 0 fi + if [[ -n "${COORD_SRUN_PID}" ]] && ! tracked_process_is_running "${COORD_SRUN_PID}"; then + wait "${COORD_SRUN_PID}" || coord_rc=$? + COORD_SRUN_PID="" + collect_startup_diagnostics "coordinator srun exited with status ${coord_rc}" + echo_error "coord did not start; its srun exited with status ${coord_rc}" + fi sleep 5 done + collect_startup_diagnostics "coordinator did not become ACTIVE" echo_error "coord did not start. state: $state" } @@ -600,15 +1459,29 @@ function wait_for_workers_to_register { [ $# -ne 1 ] && echo_error "$0 expected one argument for 'expected number of workers'" echo "waiting for $1 workers to register" local expected_num_workers=$1 - local num_workers=0 + local num_workers=0 index pid worker_rc for i in {1..60}; do - num_workers=$(curl -s http://${COORD}:${PORT}/v1/node | jq length) - if (( $num_workers == $expected_num_workers )); then + num_workers=$(curl -fsS "http://${COORD}:${PORT}/v1/node" 2>/dev/null \ + | python3 -c 'import json, sys; print(len(json.load(sys.stdin)))' 2>/dev/null \ + || echo 0) + if [[ "${num_workers}" =~ ^[0-9]+$ ]] && (( num_workers == expected_num_workers )); then echo "workers registered. num_nodes: $num_workers" return 0 fi + for ((index = 0; index < ${#WORKER_SRUN_PIDS[@]}; ++index)); do + pid="${WORKER_SRUN_PIDS[${index}]}" + [[ -n "${pid}" ]] || continue + if ! tracked_process_is_running "${pid}"; then + worker_rc=0 + wait "${pid}" || worker_rc=$? + WORKER_SRUN_PIDS[${index}]="" + collect_startup_diagnostics "worker ${WORKER_SRUN_IDS[${index}]} on ${WORKER_SRUN_NODES[${index}]} exited with status ${worker_rc}; ${num_workers}/${expected_num_workers} workers registered" + echo_error "worker ${WORKER_SRUN_IDS[${index}]} exited before cluster registration completed" + fi + done sleep 5 done + collect_startup_diagnostics "worker registration timed out at ${num_workers}/${expected_num_workers}" echo_error "workers failed to register. num_nodes: $num_workers" } @@ -630,7 +1503,7 @@ function validate_config_directory { } function collect_results { - local result_dir="${SCRIPT_DIR}/result_dir" + local result_dir="${RESULT_DIR:-${SCRIPT_DIR}/result_dir}" echo "Copying configs to ${result_dir}/configs/..." mkdir -p "${result_dir}/configs" @@ -643,7 +1516,7 @@ function collect_results { } function inject_benchmark_metadata { - local result_file="${SCRIPT_DIR}/result_dir/benchmark_result.json" + local result_file="${RESULT_DIR:-${SCRIPT_DIR}/result_dir}/benchmark_result.json" if [ ! -f "${result_file}" ]; then echo "Warning: ${result_file} not found, skipping metadata injection" return @@ -685,32 +1558,60 @@ function inject_benchmark_metadata { image_digest="${image_digest:-unknown}" echo "Image digest: ${image_digest}" - local tmp_file - tmp_file=$(mktemp) - jq --arg kind "$kind" \ - --arg timestamp "$timestamp" \ - --argjson n_workers "$NUM_WORKERS" \ - --argjson node_count "$NUM_NODES" \ - --argjson scale_factor "$SCALE_FACTOR" \ - --argjson gpu_count "$gpu_count" \ - --arg gpu_name "$gpu_name" \ - --argjson num_drivers "$num_drivers" \ - --arg worker_image "$WORKER_IMAGE" \ - --arg image_digest "$image_digest" \ - --arg engine "$engine" \ - '.context += { - kind: $kind, - timestamp: $timestamp, - n_workers: $n_workers, - node_count: $node_count, - scale_factor: $scale_factor, - gpu_count: $gpu_count, - gpu_name: $gpu_name, - num_drivers: $num_drivers, - worker_image: $worker_image, - image_digest: $image_digest, - engine: $engine - }' "${result_file}" > "${tmp_file}" && mv "${tmp_file}" "${result_file}" + python3 - "${result_file}" \ + "${kind}" "${timestamp}" "${NUM_WORKERS}" "${NUM_NODES}" \ + "${SCALE_FACTOR}" "${gpu_count}" "${gpu_name}" "${num_drivers}" \ + "${WORKER_IMAGE}" "${image_digest}" "${engine}" <<'PY' +import json +import os +import pathlib +import sys + +( + result_path, + kind, + timestamp, + n_workers, + node_count, + scale_factor, + gpu_count, + gpu_name, + num_drivers, + worker_image, + image_digest, + engine, +) = sys.argv[1:] + +path = pathlib.Path(result_path) +with path.open() as source: + result = json.load(source) + +context = result.get("context") +if not isinstance(context, dict): + context = {} + result["context"] = context +context.update( + { + "kind": kind, + "timestamp": timestamp, + "n_workers": int(n_workers), + "node_count": int(node_count), + "scale_factor": int(scale_factor), + "gpu_count": int(gpu_count), + "gpu_name": gpu_name, + "num_drivers": int(num_drivers), + "worker_image": worker_image, + "image_digest": image_digest, + "engine": engine, + } +) + +temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") +with temporary.open("w") as output: + json.dump(result, output, indent=2) + output.write("\n") +os.replace(temporary, path) +PY echo "Injected benchmark metadata into ${result_file}" } @@ -772,7 +1673,7 @@ function wait_for_nsys_report_generation { sleep 5 done - echo "Copying nsys reports to ${SCRIPT_DIR}/result_dir/..." - cp "${LOGS}"/*.nsys-rep "${SCRIPT_DIR}/result_dir/" + echo "Copying nsys reports to ${RESULT_DIR:-${SCRIPT_DIR}/result_dir}/..." + cp "${LOGS}"/*.nsys-rep "${RESULT_DIR:-${SCRIPT_DIR}/result_dir}/" fi } diff --git a/presto/slurm/presto-nvl72/launch-run.sh b/presto/slurm/presto-nvl72/launch-run.sh index 92807312..b6d63516 100755 --- a/presto/slurm/presto-nvl72/launch-run.sh +++ b/presto/slurm/presto-nvl72/launch-run.sh @@ -14,7 +14,10 @@ # [-i|--iterations ] [--cpu] [-g|--num-workers-per-node ] # [-w|--worker-image ] [-c|--coord-image ] # [-o|--output-path ] [-q|--queries ] -# [--disable-gds] [-m|--metrics] [-p|--profile] +# [--queries-file ] +# [--verify-cpu-ucx] +# [--legacy-cpu-numa] +# [--disable-gds] [-m|--metrics] [-p|--profile] [--perf] # [additional sbatch options] # ============================================================================== @@ -42,8 +45,26 @@ SCRIPT_DIR="$PWD" ENABLE_GDS=1 ENABLE_METRICS=0 ENABLE_NSYS=0 -NSYS_WORKER_ID=0 +ENABLE_PERF=0 +NSYS_WORKER_IDS="0" +PROFILE_ITERATIONS="" # comma-separated iter indices; empty = combined per query +NSYS_LAUNCH_OPTS="-t nvtx,cuda" # trace flags passed to `nsys launch` +PERF_WORKER_ID=0 +PERF_FREQUENCY="${PERF_FREQUENCY:-19}" +PERF_EVENT="${PERF_EVENT:-cpu-clock:u}" +PERF_CALL_GRAPH="${PERF_CALL_GRAPH:-dwarf,8192}" +PERF_USER_REGS="${PERF_USER_REGS:-auto}" +PERF_NOFILE_LIMIT="${PERF_NOFILE_LIMIT:-65536}" +PERF_THREADS_PER_SHARD="${PERF_THREADS_PER_SHARD:-128}" +PERF_RECORD_STOP_TIMEOUT="${PERF_RECORD_STOP_TIMEOUT:-120}" +PERF_POSTPROCESS="${PERF_POSTPROCESS:-0}" +PERF_FINALIZE_TIMEOUT="${PERF_FINALIZE_TIMEOUT:-0}" QUERIES="" +QUERIES_FILE="" +CONFIG_OVERRIDES="" +# Deliberately CLI-only. A stale exported VERIFY_CPU_UCX must not silently turn +# verification on in a later run; --verify-cpu-ucx below is the only enabler. +VERIFY_CPU_UCX=0 usage() { cat < Override coordinator image from cluster config -o, --output-path Copy results into this directory after the run -q, --queries Comma-separated query filter (e.g. "1,6,21") + --queries-file Custom JSON query file passed to run_benchmark.sh --worker-env-file Override worker.env (default: ./worker.env) + --verify-cpu-ucx Assert every CPU UCX listener is ready before + queries; for multi-worker runs, also assert that + a query uses CPU UCX replacement operators. + Enables UCX diagnostics. + --config-overrides Semicolon-separated property overrides applied + after generate_configs (e.g. + "task.max-drivers-per-task=8;cudf.batch_size_min_threshold=200000") --cpu Use CPU partition/images (overrides cluster default) --gpu Use GPU partition/images (overrides cluster default) + --numa Enable NUMA pinning for workers --no-numa Disable NUMA pinning for workers --disable-gds Disable GPU Direct Storage -m, --metrics Enable metrics collection -p, --profile Enable nsys profiling - --nsys-worker-id Worker ID to profile (default: ${NSYS_WORKER_ID}) + --perf Sample worker 0 with host perf during each + selected query/profile iteration. Captures raw + perf.data plus a portable symfs under + result_dir_/profiles/perf/worker_0/. + --perf-frequency Sampling frequency (default: ${PERF_FREQUENCY}) + --perf-event perf event (default: ${PERF_EVENT}) + --perf-call-graph perf call-graph mode (default: ${PERF_CALL_GRAPH}) + --perf-user-regs Registers passed to perf --user-regs, or + 'auto'/'perf-default' (default: ${PERF_USER_REGS}) + --perf-nofile-limit Minimum host perf soft nofile limit + (default: ${PERF_NOFILE_LIMIT}) + --perf-threads-per-shard + Maximum target TIDs assigned to one perf + recorder (default: ${PERF_THREADS_PER_SHARD}) + --perf-record-stop-timeout + Maximum time for perf record to flush and stop + after a query (default: + ${PERF_RECORD_STOP_TIMEOUT}) + --perf-postprocess Generate reports and flamegraphs after all raw + captures are published. By default this expensive + work is deferred to process-perf-captures.sh. + --perf-finalize-timeout + Maximum wait for post-suite artifact publication; + 0 uses the Slurm job time limit (default: + ${PERF_FINALIZE_TIMEOUT}) + --nsys-worker-ids Worker IDs to profile: comma list (e.g. "0,3,5") or + 'all' to profile every worker (default: ${NSYS_WORKER_IDS}) + --nsys-worker-id Alias for --nsys-worker-ids accepting a single ID + --profile-iterations + Comma-separated 0-based iteration indices to + profile separately (e.g. "1" to skip iteration + 0, or "0,1" to capture both). Applies to nsys + and perf. When unset, one capture spans every + iteration of each selected query. + --nsys-launch-opts Options passed to \`nsys launch\` controlling + what is traced. Default: "-t nvtx,cuda". + Examples: + "-t nvtx,cuda,osrt" (add OS runtime) + "-t nvtx,ucx,osrt --sample=process-tree --backtrace=dwarf" + "-t nvtx,cuda --gpu-metrics-device=all" + "-t nvtx,cuda --cuda-memory-usage=true" -h, --help Show this help message and exit Any arguments after -- are passed directly to sbatch. @@ -90,14 +160,47 @@ while [[ $# -gt 0 ]]; do -c|--coord-image) requires_value "$1" "${2:-}"; COORD_IMAGE="$2"; shift 2 ;; -o|--output-path) requires_value "$1" "${2:-}"; OUTPUT_PATH="$2"; shift 2 ;; -q|--queries) requires_value "$1" "${2:-}"; QUERIES="$2"; shift 2 ;; + --queries-file) requires_value "$1" "${2:-}"; QUERIES_FILE="$2"; shift 2 ;; --worker-env-file) requires_value "$1" "${2:-}"; WORKER_ENV_FILE="$2"; shift 2 ;; - --nsys-worker-id) requires_value "$1" "${2:-}"; NSYS_WORKER_ID="$2"; shift 2 ;; + --verify-cpu-ucx) VERIFY_CPU_UCX=1; shift ;; + --config-overrides) requires_value "$1" "${2:-}"; CONFIG_OVERRIDES="$2"; shift 2 ;; + --nsys-worker-id) requires_value "$1" "${2:-}"; NSYS_WORKER_IDS="$2"; shift 2 ;; + --nsys-worker-ids) requires_value "$1" "${2:-}"; NSYS_WORKER_IDS="$2"; shift 2 ;; + --profile-iterations) requires_value "$1" "${2:-}"; PROFILE_ITERATIONS="$2"; shift 2 ;; + --nsys-launch-opts) + # Can't use requires_value here — it disallows dash-prefixed values, + # but valid nsys flags like "-t nvtx,ucx --sample=process-tree" start with -. + [[ -n "${2:-}" ]] || { echo "Error: $1 requires a value" >&2; exit 1; } + NSYS_LAUNCH_OPTS="$2"; shift 2 ;; --cpu) VARIANT_TYPE="cpu"; shift ;; --gpu) VARIANT_TYPE="gpu"; shift ;; + --numa) USE_NUMA="1"; shift ;; --no-numa) USE_NUMA="0"; shift ;; --disable-gds) ENABLE_GDS=0; shift ;; -m|--metrics) ENABLE_METRICS=1; shift ;; -p|--profile) ENABLE_NSYS=1; shift ;; + --perf) ENABLE_PERF=1; shift ;; + --perf-frequency) requires_value "$1" "${2:-}"; PERF_FREQUENCY="$2"; shift 2 ;; + --perf-event) requires_value "$1" "${2:-}"; PERF_EVENT="$2"; shift 2 ;; + --perf-call-graph) + [[ -n "${2:-}" ]] || { echo "Error: $1 requires a value" >&2; exit 1; } + PERF_CALL_GRAPH="$2"; shift 2 ;; + --perf-user-regs) + requires_value "$1" "${2:-}" + PERF_USER_REGS="$2"; shift 2 ;; + --perf-nofile-limit) + requires_value "$1" "${2:-}" + PERF_NOFILE_LIMIT="$2"; shift 2 ;; + --perf-threads-per-shard) + requires_value "$1" "${2:-}" + PERF_THREADS_PER_SHARD="$2"; shift 2 ;; + --perf-record-stop-timeout) + requires_value "$1" "${2:-}" + PERF_RECORD_STOP_TIMEOUT="$2"; shift 2 ;; + --perf-postprocess) PERF_POSTPROCESS=1; shift ;; + --perf-finalize-timeout) + requires_value "$1" "${2:-}" + PERF_FINALIZE_TIMEOUT="$2"; shift 2 ;; -h|--help) usage; exit 0 ;; --) shift; EXTRA_ARGS+=("$@"); break ;; *) EXTRA_ARGS+=("$1"); shift ;; @@ -106,12 +209,52 @@ done [[ -z "${NODES_COUNT}" ]] && { echo "Error: -n|--nodes is required (see --help)" >&2; exit 1; } [[ -z "${SCALE_FACTOR}" ]] && { echo "Error: -s|--scale-factor is required (see --help)" >&2; exit 1; } - -# Clean up old output files — use rm -rf so subdirectories (e.g. query_results/) -# are fully removed and stale benchmark_result.json cannot survive a cancelled run. -rm -rf result_dir logs 2>/dev/null || true -rm -f *.out *.err 2>/dev/null || true -mkdir -p result_dir logs +if [[ -n "${PROFILE_ITERATIONS}" ]] && ! [[ "${PROFILE_ITERATIONS}" =~ ^[0-9]+(,[0-9]+)*$ ]]; then + echo "Error: --profile-iterations expects a comma-separated list of 0-based ints; got '${PROFILE_ITERATIONS}'" >&2 + exit 1 +fi +if [[ "${ENABLE_NSYS}" == "1" && "${ENABLE_PERF}" == "1" ]]; then + echo "Error: --profile (nsys) and --perf must run in separate jobs" >&2 + exit 1 +fi +if [[ "${PERF_POSTPROCESS}" == "1" && "${ENABLE_PERF}" != "1" ]]; then + echo "Error: --perf-postprocess requires --perf" >&2 + exit 1 +fi +if ! [[ "${PERF_FREQUENCY}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: --perf-frequency must be a positive integer; got '${PERF_FREQUENCY}'" >&2 + exit 1 +fi +if ! [[ "${PERF_NOFILE_LIMIT}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: --perf-nofile-limit must be a positive integer; got '${PERF_NOFILE_LIMIT}'" >&2 + exit 1 +fi +if ! [[ "${PERF_THREADS_PER_SHARD}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: --perf-threads-per-shard must be a positive integer; got '${PERF_THREADS_PER_SHARD}'" >&2 + exit 1 +fi +if ! [[ "${PERF_RECORD_STOP_TIMEOUT}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: --perf-record-stop-timeout must be a positive integer; got '${PERF_RECORD_STOP_TIMEOUT}'" >&2 + exit 1 +fi +if ! [[ "${PERF_POSTPROCESS}" =~ ^[01]$ ]]; then + echo "Error: PERF_POSTPROCESS must be 0 or 1; got '${PERF_POSTPROCESS}'" >&2 + exit 1 +fi +if ! [[ "${PERF_FINALIZE_TIMEOUT}" =~ ^[0-9]+$ ]]; then + echo "Error: --perf-finalize-timeout must be a non-negative integer; got '${PERF_FINALIZE_TIMEOUT}'" >&2 + exit 1 +fi +if [[ -n "${PROFILE_ITERATIONS}" ]]; then + IFS=',' read -ra _profile_iteration_list <<< "${PROFILE_ITERATIONS}" + for _profile_iteration in "${_profile_iteration_list[@]}"; do + if (( _profile_iteration >= NUM_ITERATIONS )); then + echo "Error: profile iteration ${_profile_iteration} is outside -i ${NUM_ITERATIONS} (indices are 0-based)" >&2 + exit 1 + fi + done + unset _profile_iteration _profile_iteration_list +fi echo "Submitting Presto TPC-H benchmark job..." echo "" @@ -124,6 +267,27 @@ resolve_cluster_variant "${VARIANT_TYPE}" : "${NUM_GPUS_PER_NODE:=${CLUSTER_NUM_WORKERS_PER_NODE:-}}" : "${USE_NUMA:=${CLUSTER_USE_NUMA:-0}}" +# GDS is a GPU-only data path. CPU launches should not require --disable-gds +# and should never expose cufile settings to a CPU worker image. +if [[ "${VARIANT_TYPE}" == "cpu" ]]; then + ENABLE_GDS=0 +fi + +if [[ "${VERIFY_CPU_UCX}" == "1" ]]; then + [[ "${VARIANT_TYPE}" == "cpu" ]] || { echo "Error: --verify-cpu-ucx requires --cpu" >&2; exit 1; } + export UCX_LOG_LEVEL="${UCX_LOG_LEVEL:-info}" + export UCX_PROTO_INFO="${UCX_PROTO_INFO:-y}" +fi + +# Expand --nsys-worker-ids=all to the full 0..N-1 range now that NUM_GPUS_PER_NODE +# is known. Done here (not at arg-parse time) because total worker count depends +# on cluster-resolved values. +if [[ "${NSYS_WORKER_IDS}" == "all" ]]; then + _total_workers=$(( NODES_COUNT * NUM_GPUS_PER_NODE )) + NSYS_WORKER_IDS="$(seq -s, 0 $(( _total_workers - 1 )))" + unset _total_workers +fi + # Validate required values before submitting VTYPE_UPPER="${VARIANT_TYPE^^}" [[ -z "${WORKER_IMAGE}" ]] && { echo "Error: worker image not set — set CLUSTER_${VTYPE_UPPER}_DEFAULT_WORKER_IMAGE in cluster_config.env or pass -w"; exit 1; } @@ -138,6 +302,25 @@ build_cluster_sbatch_args "${CLUSTER_TIME_BENCHMARK}" # Pre-flight: verify prerequisites before queueing the job. ANALYZE_HINT="./launch-analyze-tables.sh -s ${SCALE_FACTOR}" +preflight_file "${WORKER_ENV_FILE}" "worker environment" \ + "Set WORKER_ENV_FILE or pass --worker-env-file " +WORKER_ENV_FILE="$(canonicalize_file_path "${WORKER_ENV_FILE}")" + +if [[ -n "${QUERIES_FILE}" ]]; then + preflight_file "${QUERIES_FILE}" "queries JSON" + QUERIES_FILE="$(canonicalize_file_path "${QUERIES_FILE}")" + if [[ "${QUERIES_FILE}" != "${VT_ROOT}/"* ]]; then + echo "Error: --queries-file must be inside ${VT_ROOT} so it is available in the CLI container." >&2 + echo " Got: ${QUERIES_FILE}" >&2 + exit 1 + fi + if ! python3 -m json.tool "${QUERIES_FILE}" >/dev/null 2>&1; then + echo "Error: queries file is not valid JSON: ${QUERIES_FILE}" >&2 + exit 1 + fi + QUERIES_FILE="/workspace/${QUERIES_FILE#"${VT_ROOT}/"}" +fi +preflight_image_roles "${WORKER_IMAGE}" "${COORD_IMAGE}" preflight_image "${WORKER_IMAGE}" \ "Pull it (see ./pull_ghcr_image.sh) or override with -w " preflight_image "${COORD_IMAGE}" \ @@ -146,6 +329,13 @@ preflight_dir "${DATA}/tpch-rs-${SCALE_FACTOR}" "TPC-H SF${SCALE_FACTOR} data" \ "./launch-gen-data.sh -s ${SCALE_FACTOR} -o ${DATA}/tpch-rs-${SCALE_FACTOR}" preflight_metastore "${SCALE_FACTOR}" "${ANALYZE_HINT}" +# Only transient live logs are reused. Completed result directories are named +# with their Slurm job ID and are never removed by a later launch, so an scp or +# rsync of an earlier run cannot race this cleanup. +rm -rf logs 2>/dev/null || true +rm -f *.out *.err 2>/dev/null || true +mkdir -p logs + # Submit job (include nodes/SF/iterations in file names) OUT_FMT="logs/presto-tpch-run_n${NODES_COUNT}_sf${SCALE_FACTOR}_i${NUM_ITERATIONS}_%j.out" ERR_FMT="logs/presto-tpch-run_n${NODES_COUNT}_sf${SCALE_FACTOR}_i${NUM_ITERATIONS}_%j.err" @@ -162,23 +352,61 @@ GRES_OPT=$([[ "$VARIANT_TYPE" == "gpu" ]] && echo "--gres=gpu:${NUM_GPUS_PER_NOD build_common_export_vars EXPORT_VARS+=",NUM_ITERATIONS=${NUM_ITERATIONS}" EXPORT_VARS+=",ENABLE_GDS=${ENABLE_GDS},ENABLE_METRICS=${ENABLE_METRICS}" -EXPORT_VARS+=",ENABLE_NSYS=${ENABLE_NSYS},NSYS_WORKER_ID=${NSYS_WORKER_ID}" -# Comma-separated query list can't ride EXPORT_VARS (comma is the separator); -# export it so sbatch picks it up via the ALL inheritance. +EXPORT_VARS+=",ENABLE_NSYS=${ENABLE_NSYS}" +EXPORT_VARS+=",ENABLE_PERF=${ENABLE_PERF},PERF_WORKER_ID=${PERF_WORKER_ID}" +EXPORT_VARS+=",VERIFY_CPU_UCX=${VERIFY_CPU_UCX}" +# Comma-separated values can't ride EXPORT_VARS (comma is the sbatch --export +# field separator); export them so sbatch picks them up via the ALL inheritance. [[ -n "${QUERIES}" ]] && export QUERIES +[[ -n "${QUERIES_FILE}" ]] && export QUERIES_FILE +export NSYS_WORKER_IDS +[[ -n "${PROFILE_ITERATIONS}" ]] && export PROFILE_ITERATIONS +# NSYS_LAUNCH_OPTS may contain spaces / equals signs / commas — ride ALL +# inheritance rather than EXPORT_VARS so the whole quoted string survives. +export NSYS_LAUNCH_OPTS +# PERF_CALL_GRAPH and PERF_USER_REGS contain commas, and the remaining values +# may contain perf punctuation. Inherit them through --export=ALL instead of +# splitting the sbatch export list. +export PERF_FREQUENCY PERF_EVENT PERF_CALL_GRAPH PERF_USER_REGS +export PERF_NOFILE_LIMIT PERF_THREADS_PER_SHARD PERF_RECORD_STOP_TIMEOUT +export PERF_POSTPROCESS PERF_FINALIZE_TIMEOUT +# CONFIG_OVERRIDES uses ';' internally but values may contain commas (e.g. +# "32MB,64MB" would break, but more pressingly any later additions); easiest +# to keep it off EXPORT_VARS entirely. +[[ -n "${CONFIG_OVERRIDES}" ]] && export CONFIG_OVERRIDES JOB_ID=$(sbatch --job-name="${JOB_NAME}" --nodes="${NODES_COUNT}" "${NODELIST_ARG[@]}" \ "${CLUSTER_SBATCH_ARGS[@]}" \ --export="${EXPORT_VARS}" \ --output="${OUT_FMT}" --error="${ERR_FMT}" "${EXTRA_ARGS[@]}" ${GRES_OPT} \ -run-presto-benchmarks.slurm | awk '{print $NF}') + run-presto-benchmarks.slurm | awk '{print $NF}') +RUN_RESULT_DIR="result_dir_${JOB_ID}" OUT_FILE="${OUT_FMT//%j/${JOB_ID}}" ERR_FILE="${ERR_FMT//%j/${JOB_ID}}" +# A text pointer is safe for automation: callers resolve it once and then copy +# the immutable job-specific directory. Unlike a mutable result_dir symlink, it +# cannot redirect individual files to a newer run midway through rsync/scp. +LATEST_RESULT_POINTER="${SCRIPT_DIR}/latest_result_dir.txt" +LATEST_RESULT_POINTER_TMP="${LATEST_RESULT_POINTER}.tmp.$$" +printf '%s\n' "${RUN_RESULT_DIR}" > "${LATEST_RESULT_POINTER_TMP}" +mv -f "${LATEST_RESULT_POINTER_TMP}" "${LATEST_RESULT_POINTER}" + +echo "Job submitted with ID: $JOB_ID" +echo "" + # Resolve and print first node IP once nodes are allocated echo "Resolving first node IP..." for i in {1..60}; do STATE=$(squeue -j "$JOB_ID" -h -o "%T" 2>/dev/null || true) NODELIST=$(squeue -j "$JOB_ID" -h -o "%N" 2>/dev/null || true) + if [[ -z "${STATE:-}" ]]; then + echo "Job ${JOB_ID} is no longer in squeue before node allocation." + break + fi + if [[ "${STATE}" =~ ^(CANCELLED|FAILED|TIMEOUT|COMPLETED|NODE_FAIL|PREEMPTED|BOOT_FAIL|DEADLINE|OUT_OF_MEMORY)$ ]]; then + echo "Job ${JOB_ID} reached state ${STATE} before node allocation." + break + fi if [[ -n "${NODELIST:-}" && "${NODELIST}" != "(null)" ]]; then FIRST_NODE=$(scontrol show hostnames "$NODELIST" | head -n 1) if [[ -n "${FIRST_NODE:-}" ]]; then @@ -198,8 +426,6 @@ for i in {1..60}; do sleep 5 done -echo "Job submitted with ID: $JOB_ID" -echo "" print_monitor_hints "${JOB_ID}" "${OUT_FILE}" "${ERR_FILE}" \ "tail -f logs/coord.log" \ "tail -f logs/worker_*.log" \ @@ -212,12 +438,21 @@ echo "" echo "Output files:" ls -lh "${OUT_FILE}" "${ERR_FILE}" 2>/dev/null || echo "No output files found" show_job_output "${OUT_FILE}" "${ERR_FILE}" "logs/cli.log" "benchmark results" -[[ "${JOB_STATE}" == "COMPLETED" ]] || exit 1 -if [[ -n "${OUTPUT_PATH}" ]]; then +if [[ -d "${RUN_RESULT_DIR}" ]]; then + echo "" + echo "Job results preserved at: ${SCRIPT_DIR}/${RUN_RESULT_DIR}" + echo "Latest-result pointer: ${LATEST_RESULT_POINTER}" +else + echo "No job result directory was created at ${SCRIPT_DIR}/${RUN_RESULT_DIR}" >&2 +fi + +if [[ -n "${OUTPUT_PATH}" && -d "${RUN_RESULT_DIR}" ]]; then echo "" echo "Copying results to ${OUTPUT_PATH}..." mkdir -p "${OUTPUT_PATH}" - cp -r result_dir/. "${OUTPUT_PATH}/" + cp -r "${RUN_RESULT_DIR}/." "${OUTPUT_PATH}/" echo "Results copied to ${OUTPUT_PATH}" fi + +[[ "${JOB_STATE}" == "COMPLETED" ]] || exit 1 diff --git a/presto/slurm/presto-nvl72/perf-record-sharded.sh b/presto/slurm/presto-nvl72/perf-record-sharded.sh new file mode 100755 index 00000000..f8edeac5 --- /dev/null +++ b/presto/slurm/presto-nvl72/perf-record-sharded.sh @@ -0,0 +1,625 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Run one logical perf capture as several independently limited perf record +# processes. Some Arm server perf builds expand even a software event over the +# PMU CPU map for every selected thread. A warmed Presto worker can therefore +# exceed RLIMIT_NOFILE in one perf process. Each shard targets a disjoint set of +# existing TIDs and enables inheritance, so the union covers the complete +# worker while every recorder stays comfortably below its descriptor ceiling. + +set -uo pipefail + +target_pid="" +output_dir="" +diagnostic_dir="" +control_fifo="" +ack_fifo="" +event="cpu-clock:u" +frequency=19 +call_graph="dwarf,8192" +user_regs="" +threads_per_shard=128 +control_ack_timeout=120 +record_stop_timeout=120 +startup_attempts=3 +startup_ready_timeout=30 + +usage() { + cat <&2; usage >&2; exit 2 ;; + esac +done + +[[ "${target_pid}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid --target-pid: ${target_pid}" >&2; exit 2; } +[[ "${frequency}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid --frequency: ${frequency}" >&2; exit 2; } +[[ "${threads_per_shard}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid --threads-per-shard: ${threads_per_shard}" >&2; exit 2; } +[[ "${control_ack_timeout}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid --control-ack-timeout: ${control_ack_timeout}" >&2; exit 2; } +[[ "${record_stop_timeout}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid --record-stop-timeout: ${record_stop_timeout}" >&2; exit 2; } +[[ "${startup_attempts}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid --startup-attempts: ${startup_attempts}" >&2; exit 2; } +[[ "${startup_ready_timeout}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid --startup-ready-timeout: ${startup_ready_timeout}" >&2; exit 2; } +[[ -n "${output_dir}" ]] || { echo "Error: --output-dir is required" >&2; exit 2; } +[[ -n "${diagnostic_dir}" ]] || { echo "Error: --diagnostic-dir is required" >&2; exit 2; } +[[ -p "${control_fifo}" ]] || { echo "Error: control FIFO not found: ${control_fifo}" >&2; exit 2; } +[[ -p "${ack_fifo}" ]] || { echo "Error: acknowledgement FIFO not found: ${ack_fifo}" >&2; exit 2; } +kill -0 "${target_pid}" 2>/dev/null || { echo "Error: target PID ${target_pid} is not alive" >&2; exit 1; } + +mkdir -p "${output_dir}" "${diagnostic_dir}" + +declare -a target_tids=() +for task_dir in "/proc/${target_pid}/task/"*; do + tid="${task_dir##*/}" + [[ "${tid}" =~ ^[1-9][0-9]*$ ]] && target_tids+=("${tid}") +done +if (( ${#target_tids[@]} == 0 )); then + echo "Error: target PID ${target_pid} has no visible threads" >&2 + exit 1 +fi +mapfile -t target_tids < <(printf '%s\n' "${target_tids[@]}" | sort -n) +printf '%s\n' "${target_tids[@]}" > "${diagnostic_dir}/perf-target-tids.txt" + +declare -a perf_sampling_args=( + -e "${event}" + -F "${frequency}" + -g + --call-graph "${call_graph}" +) +if [[ -n "${user_regs}" ]]; then + perf_sampling_args+=("--user-regs=${user_regs}") +fi + +declare -a child_pids=() +declare -a child_control_fds=() +declare -a child_ack_fds=() +declare -a child_control_fifos=() +declare -a child_ack_fifos=() +declare -a child_data_files=() +declare -a child_thread_counts=() +declare -a child_first_tids=() +declare -a child_last_tids=() +declare -a child_exit_rcs=() +declare -a child_enabled=() +declare -a child_open_fds_at_enable=() + +outer_control_fd="" +outer_ack_fd="" +cleanup_started=0 +normal_shutdown=0 + +log() { + printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" +} + +child_is_running() { + local -r index="$1" + local -r pid="${child_pids[index]:-}" + local state="" + [[ "${pid}" =~ ^[1-9][0-9]*$ ]] || return 1 + kill -0 "${pid}" 2>/dev/null || return 1 + state="$(awk '{ print $3 }' "/proc/${pid}/stat" 2>/dev/null || true)" + [[ "${state}" != "Z" ]] +} + +record_child_exit() { + local -r index="$1" + local rc=0 + [[ -z "${child_exit_rcs[index]:-}" ]] || return 0 + if [[ ! "${child_pids[index]:-}" =~ ^[1-9][0-9]*$ ]]; then + child_exit_rcs[index]="not_started" + return 0 + fi + wait "${child_pids[index]}" 2>/dev/null || rc=$? + child_exit_rcs[index]="${rc}" +} + +close_child_control() { + local -r index="$1" + local fd="" + fd="${child_ack_fds[index]:-}" + if [[ "${fd}" =~ ^[0-9]+$ ]]; then + exec {fd}>&- + fi + fd="${child_control_fds[index]:-}" + if [[ "${fd}" =~ ^[0-9]+$ ]]; then + exec {fd}>&- + fi + child_ack_fds[index]="" + child_control_fds[index]="" + [[ -z "${child_ack_fifos[index]:-}" ]] || rm -f "${child_ack_fifos[index]}" + [[ -z "${child_control_fifos[index]:-}" ]] || rm -f "${child_control_fifos[index]}" +} + +write_manifest() { + local -r phase="$1" + local -r manifest="${diagnostic_dir}/perf-shards.tsv" + local index="" + local pid="" + local state="" + local open_fds="" + local open_fds_at_enable="" + local data_bytes="" + + printf 'shard\tphase\tperf_pid\tthread_count\tfirst_tid\tlast_tid\tcurrent_open_fds\topen_fds_at_enable\texit_status\tdata_file\tdata_bytes\n' > "${manifest}" + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + pid="${child_pids[index]}" + if child_is_running "${index}"; then + state="running" + open_fds="$(find "/proc/${pid}/fd" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)" + else + record_child_exit "${index}" + state="${child_exit_rcs[index]:-unknown}" + open_fds="0" + fi + if [[ "${phase}" == "enabled" ]]; then + child_open_fds_at_enable[index]="${open_fds}" + fi + open_fds_at_enable="${child_open_fds_at_enable[index]:-unavailable}" + if [[ -s "${child_data_files[index]}" ]]; then + data_bytes="$(stat -c '%s' "${child_data_files[index]}")" + else + data_bytes="0" + fi + printf '%03d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${index}" \ + "${phase}" \ + "${pid}" \ + "${child_thread_counts[index]}" \ + "${child_first_tids[index]}" \ + "${child_last_tids[index]}" \ + "${open_fds}" \ + "${open_fds_at_enable}" \ + "${state}" \ + "$(basename -- "${child_data_files[index]}")" \ + "${data_bytes}" >> "${manifest}" + done +} + +await_child_ack() { + local -r index="$1" + local -r command="$2" + local -r timeout_seconds="${3:-${control_ack_timeout}}" + local response="" + local tick=0 + local -r fd="${child_ack_fds[index]}" + + for ((tick = 0; tick < timeout_seconds * 5; ++tick)); do + if IFS= read -r -t 0.2 -u "${fd}" response; then + [[ "${response}" == "ack" ]] && return 0 + log "shard ${index}: ignoring unexpected control response '${response}' for '${command}'" + fi + if ! child_is_running "${index}"; then + record_child_exit "${index}" + return 1 + fi + done + return 2 +} + +terminate_child() { + local -r index="$1" + local elapsed=0 + + if child_is_running "${index}"; then + kill -INT "${child_pids[index]}" 2>/dev/null || true + fi + while child_is_running "${index}" && (( elapsed < 50 )); do + sleep 0.2 + ((elapsed += 1)) + done + if child_is_running "${index}"; then + kill -TERM "${child_pids[index]}" 2>/dev/null || true + sleep 1 + fi + if child_is_running "${index}"; then + kill -KILL "${child_pids[index]}" 2>/dev/null || true + fi + record_child_exit "${index}" +} + +launch_child_ready() { + local -r index="$1" + local -r requested_tid_csv="$2" + local -r data_file="$3" + local -r child_control_fifo="$4" + local -r child_ack_fifo="$5" + local -r stdout_file="$6" + local -r stderr_file="$7" + local attempt=0 + local child_control_fd="" + local child_ack_fd="" + local failure_stdout="" + local failure_stderr="" + local data_bytes=0 + local dropped_count=0 + local live_tid_csv="" + local ready_rc=0 + local requested_count=0 + local shard_id="" + local tid="" + local -a dropped_tids=() + local -a live_tids=() + local -a perf_verbosity_args=() + local -a requested_tids=() + + printf -v shard_id '%03d' "${index}" + IFS=',' read -r -a requested_tids <<< "${requested_tid_csv}" + requested_count="${#requested_tids[@]}" + for ((attempt = 1; attempt <= startup_attempts; ++attempt)); do + # Worker pools may retire a thread while the recorders are being opened + # serially. perf aborts the whole -t list on ESRCH, so rebuild this shard + # from its original, disjoint TID set before every attempt. New worker + # threads remain covered through --inherit; never move a TID between shards. + live_tids=() + dropped_tids=() + for tid in "${requested_tids[@]}"; do + if [[ -d "/proc/${target_pid}/task/${tid}" ]]; then + live_tids+=("${tid}") + else + dropped_tids+=("${tid}") + printf '%03d\t%d\t%s\texited_before_attach\n' \ + "${index}" "${attempt}" "${tid}" \ + >> "${diagnostic_dir}/perf-dropped-tids.tsv" + fi + done + dropped_count="${#dropped_tids[@]}" + child_thread_counts[index]="${#live_tids[@]}" + child_data_files[index]="${data_file}" + if (( ${#live_tids[@]} == 0 )); then + child_first_tids[index]="unavailable" + child_last_tids[index]="unavailable" + child_pids[index]="unavailable" + child_exit_rcs[index]="not_started" + printf '%03d\t%d\tfailed\tunavailable\t%s\t0\t%s\t0\tno_live_tids\t0\n' \ + "${index}" "${attempt}" "${requested_count}" "${dropped_count}" \ + >> "${diagnostic_dir}/perf-startup-attempts.tsv" + echo "Error: all ${requested_count} snapshotted TIDs in perf shard ${index} exited before attachment" >&2 + return 1 + fi + child_first_tids[index]="${live_tids[0]}" + child_last_tids[index]="${live_tids[${#live_tids[@]} - 1]}" + live_tid_csv="$(IFS=,; printf '%s' "${live_tids[*]}")" + + rm -f "${data_file}" "${child_control_fifo}" "${child_ack_fifo}" + if ! mkfifo "${child_control_fifo}" "${child_ack_fifo}"; then + echo "Error: could not create control FIFOs for perf shard ${index}" >&2 + return 1 + fi + child_control_fd="" + child_ack_fd="" + if ! exec {child_control_fd}<>"${child_control_fifo}"; then + echo "Error: could not open control FIFO for perf shard ${index}" >&2 + return 1 + fi + if ! exec {child_ack_fd}<>"${child_ack_fifo}"; then + echo "Error: could not open acknowledgement FIFO for perf shard ${index}" >&2 + exec {child_control_fd}>&- + return 1 + fi + + # Successful verbose perf processes emit millions of overlapping-map + # messages on this worker. Keep normal attempts quiet; if prior attempts + # could not recover, make only the final attempt verbose for diagnostics. + perf_verbosity_args=() + if (( attempt > 1 && attempt == startup_attempts )); then + perf_verbosity_args=(-v) + fi + perf record \ + "${perf_verbosity_args[@]}" \ + --buildid-all \ + --delay=-1 \ + --control "fifo:${child_control_fifo},${child_ack_fifo}" \ + "${perf_sampling_args[@]}" \ + --inherit \ + -t "${live_tid_csv}" \ + -o "${data_file}" \ + >"${stdout_file}" \ + 2>"${stderr_file}" & + + child_pids[index]=$! + child_control_fds[index]="${child_control_fd}" + child_ack_fds[index]="${child_ack_fd}" + child_control_fifos[index]="${child_control_fifo}" + child_ack_fifos[index]="${child_ack_fifo}" + child_data_files[index]="${data_file}" + child_exit_rcs[index]="" + child_enabled[index]=0 + child_open_fds_at_enable[index]="" + + # perf accepts control input before it has finished opening events. A ping + # acknowledgement is the supported readiness barrier. Starting recorders + # serially avoids a burst of hundreds of thousands of concurrent + # perf_event_open calls on a warmed worker. + ready_rc=0 + if ! printf 'ping\n' >&"${child_control_fds[index]}"; then + ready_rc=1 + else + await_child_ack "${index}" "startup ping" "${startup_ready_timeout}" + ready_rc=$? + fi + if (( ready_rc == 0 )); then + printf '%s\n' "${live_tids[@]}" \ + > "${diagnostic_dir}/perf-attached-tids.part${shard_id}.txt" + printf '%03d\t%d\tready\t%s\t%s\t%s\t%s\t%s\trunning\t%s\n' \ + "${index}" "${attempt}" "${child_pids[index]}" \ + "${requested_count}" "${#live_tids[@]}" "${dropped_count}" \ + "$(find "/proc/${child_pids[index]}/fd" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)" \ + "$(stat -c '%s' "${data_file}" 2>/dev/null || echo 0)" \ + >> "${diagnostic_dir}/perf-startup-attempts.tsv" + log "shard ${index}/${shard_count} ready on startup attempt ${attempt}" + return 0 + fi + + terminate_child "${index}" + data_bytes="$(stat -c '%s' "${data_file}" 2>/dev/null || echo 0)" + printf '%03d\t%d\tfailed\t%s\t%s\t%s\t%s\t0\t%s\t%s\n' \ + "${index}" "${attempt}" "${child_pids[index]}" \ + "${requested_count}" "${#live_tids[@]}" "${dropped_count}" \ + "${child_exit_rcs[index]:-unknown}" "${data_bytes}" \ + >> "${diagnostic_dir}/perf-startup-attempts.tsv" + close_child_control "${index}" + + if (( attempt < startup_attempts )); then + printf -v failure_stdout '%s/perf-record.part%s.attempt%03d.failed.stdout.txt' \ + "${diagnostic_dir}" "${shard_id}" "${attempt}" + printf -v failure_stderr '%s/perf-record.part%s.attempt%03d.failed.stderr.txt' \ + "${diagnostic_dir}" "${shard_id}" "${attempt}" + mv -f -- "${stdout_file}" "${failure_stdout}" 2>/dev/null || true + mv -f -- "${stderr_file}" "${failure_stderr}" 2>/dev/null || true + rm -f "${data_file}" + log "shard ${index} startup attempt ${attempt} failed with status ${child_exit_rcs[index]:-unknown}; retrying" + sleep 1 + fi + done + + echo "Error: perf shard ${index} failed all ${startup_attempts} startup attempts; see ${stderr_file}" >&2 + return 1 +} + +send_all_child_controls() { + local -r command="$1" + local index="" + local failed=0 + + # Send every command first so a slow shard cannot serialize attachment or + # finalization of the remaining recorders. + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + if ! child_is_running "${index}"; then + record_child_exit "${index}" + echo "Error: perf shard ${index} exited with status ${child_exit_rcs[index]:-unknown} before '${command}'" >&2 + failed=1 + continue + fi + if ! printf '%s\n' "${command}" >&"${child_control_fds[index]}"; then + echo "Error: could not send '${command}' to perf shard ${index}" >&2 + failed=1 + fi + done + (( failed == 0 )) || return 1 + + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + if ! await_child_ack "${index}" "${command}"; then + echo "Error: perf shard ${index} did not acknowledge '${command}'; see ${diagnostic_dir}/perf-record.part$(printf '%03d' "${index}").stderr.txt" >&2 + failed=1 + fi + done + (( failed == 0 )) +} + +wait_for_children() { + local elapsed=0 + local index="" + local running=0 + local failed=0 + + while (( elapsed < record_stop_timeout * 5 )); do + running=0 + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + if child_is_running "${index}"; then + running=1 + elif [[ -z "${child_exit_rcs[index]:-}" ]]; then + record_child_exit "${index}" + fi + done + (( running == 0 )) && break + sleep 0.2 + ((elapsed += 1)) + done + + if (( running != 0 )); then + echo "Error: one or more perf shards did not stop within ${record_stop_timeout}s; terminating them" >&2 + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + child_is_running "${index}" && kill -TERM "${child_pids[index]}" 2>/dev/null || true + done + sleep 2 + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + child_is_running "${index}" && kill -KILL "${child_pids[index]}" 2>/dev/null || true + done + failed=1 + fi + + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + record_child_exit "${index}" + if [[ "${child_exit_rcs[index]}" != "0" && "${child_exit_rcs[index]}" != "130" ]]; then + echo "Error: perf shard ${index} exited with status ${child_exit_rcs[index]}" >&2 + failed=1 + fi + done + (( failed == 0 )) +} + +terminate_children() { + local index="" + local elapsed=0 + local running=0 + + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + child_is_running "${index}" && kill -INT "${child_pids[index]}" 2>/dev/null || true + done + while (( elapsed < record_stop_timeout * 5 )); do + running=0 + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + child_is_running "${index}" && running=1 + done + (( running == 0 )) && break + sleep 0.2 + ((elapsed += 1)) + done + if (( running != 0 )); then + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + child_is_running "${index}" && kill -TERM "${child_pids[index]}" 2>/dev/null || true + done + sleep 2 + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + child_is_running "${index}" && kill -KILL "${child_pids[index]}" 2>/dev/null || true + done + fi + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + [[ -n "${child_exit_rcs[index]:-}" ]] || record_child_exit "${index}" + done +} + +cleanup() { + local index="" + local fd="" + (( cleanup_started == 0 )) || return 0 + cleanup_started=1 + if (( normal_shutdown == 0 )); then + terminate_children + fi + for ((index = 0; index < ${#child_pids[@]}; ++index)); do + close_child_control "${index}" + done + fd="${outer_ack_fd}" + if [[ "${fd}" =~ ^[0-9]+$ ]]; then + exec {fd}>&- + fi + fd="${outer_control_fd}" + if [[ "${fd}" =~ ^[0-9]+$ ]]; then + exec {fd}>&- + fi +} +trap cleanup EXIT +trap 'exit 130' INT TERM + +if ! exec {outer_control_fd}<>"${control_fifo}"; then + echo "Error: could not open outer control FIFO ${control_fifo}" >&2 + exit 1 +fi +if ! exec {outer_ack_fd}<>"${ack_fifo}"; then + echo "Error: could not open outer acknowledgement FIFO ${ack_fifo}" >&2 + exit 1 +fi + +shard_count="$(( (${#target_tids[@]} + threads_per_shard - 1) / threads_per_shard ))" +log "snapshot ${#target_tids[@]} target threads; starting ${shard_count} perf shard(s) with at most ${threads_per_shard} TIDs each" +printf 'shard\tattempt\tstatus\tperf_pid\trequested_thread_count\tlive_thread_count\tdropped_thread_count\tcurrent_open_fds\texit_status\tdata_bytes\n' \ + > "${diagnostic_dir}/perf-startup-attempts.tsv" +printf 'shard\tattempt\ttid\treason\n' \ + > "${diagnostic_dir}/perf-dropped-tids.tsv" + +for ((shard = 0; shard < shard_count; ++shard)); do + offset="$(( shard * threads_per_shard ))" + shard_tids=("${target_tids[@]:offset:threads_per_shard}") + tid_csv="$(IFS=,; echo "${shard_tids[*]}")" + shard_id="$(printf '%03d' "${shard}")" + data_file="${output_dir}/perf.data.part${shard_id}" + child_control_fifo="${output_dir}/.control.part${shard_id}" + child_ack_fifo="${output_dir}/.ack.part${shard_id}" + stdout_file="${diagnostic_dir}/perf-record.part${shard_id}.stdout.txt" + stderr_file="${diagnostic_dir}/perf-record.part${shard_id}.stderr.txt" + + if ! launch_child_ready \ + "${shard}" \ + "${tid_csv}" \ + "${data_file}" \ + "${child_control_fifo}" \ + "${child_ack_fifo}" \ + "${stdout_file}" \ + "${stderr_file}"; then + write_manifest "startup_failed" + exit 255 + fi +done + +write_manifest "ready" + +while true; do + outer_command="" + if IFS= read -r -t 0.2 -u "${outer_control_fd}" outer_command; then + case "${outer_command}" in + enable) + if ! send_all_child_controls "enable"; then + write_manifest "enable_failed" + exit 255 + fi + for ((shard = 0; shard < shard_count; ++shard)); do + child_enabled[shard]=1 + done + write_manifest "enabled" + printf 'ack\n' >&"${outer_ack_fd}" + log "all ${shard_count} perf shards enabled" + ;; + stop) + if ! send_all_child_controls "stop"; then + write_manifest "stop_failed" + exit 255 + fi + printf 'ack\n' >&"${outer_ack_fd}" + wait_rc=0 + wait_for_children || wait_rc=$? + write_manifest "stopped" + normal_shutdown=1 + exit "${wait_rc}" + ;; + ping) + printf 'ack\n' >&"${outer_ack_fd}" + ;; + *) + log "ignoring unexpected outer control command '${outer_command}'" + ;; + esac + fi + + for ((shard = 0; shard < shard_count; ++shard)); do + if ! child_is_running "${shard}"; then + record_child_exit "${shard}" + echo "Error: perf shard ${shard} exited unexpectedly with status ${child_exit_rcs[shard]}; see ${diagnostic_dir}/perf-record.part$(printf '%03d' "${shard}").stderr.txt" >&2 + write_manifest "unexpected_exit" + exit 255 + fi + done +done diff --git a/presto/slurm/presto-nvl72/perf-worker-sampler.sh b/presto/slurm/presto-nvl72/perf-worker-sampler.sh new file mode 100755 index 00000000..42d66b31 --- /dev/null +++ b/presto/slurm/presto-nvl72/perf-worker-sampler.sh @@ -0,0 +1,839 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Runs directly on the worker's compute host (outside Enroot) and attaches the +# host's kernel-matched perf binary to a presto_server PID published by the +# worker container. Pyxis does not create a PID namespace for these workers, so +# the PID is identical inside and outside the container. Query start/stop is +# controlled by token files written by perf_profiler_functions.sh. + +set -uo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd -P)" +SHARDED_RECORDER="${SCRIPT_DIR}/perf-record-sharded.sh" + +worker_id=0 +control_dir="" +output_dir="" +postprocess_script="" +postprocess=0 +event="cpu-clock:u" +frequency=19 +call_graph="dwarf,8192" +user_regs="auto" +nofile_limit=65536 +threads_per_shard=128 +pid_wait_timeout=180 +record_stop_timeout=120 +control_ack_timeout="${PERF_CONTROL_ACK_TIMEOUT:-120}" +startup_attempts="${PERF_STARTUP_ATTEMPTS:-3}" +startup_ready_timeout="${PERF_STARTUP_READY_TIMEOUT:-30}" + +usage() { + cat <&2; usage >&2; exit 2 ;; + esac +done + +[[ "${worker_id}" =~ ^[0-9]+$ ]] || { echo "Error: invalid worker ID: ${worker_id}" >&2; exit 2; } +[[ "${frequency}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid frequency: ${frequency}" >&2; exit 2; } +[[ "${nofile_limit}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid nofile limit: ${nofile_limit}" >&2; exit 2; } +[[ "${threads_per_shard}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid threads per shard: ${threads_per_shard}" >&2; exit 2; } +[[ "${pid_wait_timeout}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid PID wait timeout: ${pid_wait_timeout}" >&2; exit 2; } +[[ "${record_stop_timeout}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: invalid perf record stop timeout: ${record_stop_timeout}" >&2; exit 2; } +[[ "${control_ack_timeout}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: PERF_CONTROL_ACK_TIMEOUT must be a positive integer; got ${control_ack_timeout}" >&2; exit 2; } +[[ "${startup_attempts}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: PERF_STARTUP_ATTEMPTS must be a positive integer; got ${startup_attempts}" >&2; exit 2; } +[[ "${startup_ready_timeout}" =~ ^[1-9][0-9]*$ ]] || { echo "Error: PERF_STARTUP_READY_TIMEOUT must be a positive integer; got ${startup_ready_timeout}" >&2; exit 2; } +[[ "${postprocess}" =~ ^[01]$ ]] || { echo "Error: --postprocess must be 0 or 1; got ${postprocess}" >&2; exit 2; } +[[ -n "${control_dir}" ]] || { echo "Error: --control-dir is required" >&2; exit 2; } +[[ -n "${output_dir}" ]] || { echo "Error: --output-dir is required" >&2; exit 2; } +[[ -f "${postprocess_script}" ]] || { echo "Error: perf postprocessor not found: ${postprocess_script}" >&2; exit 2; } +[[ -f "${SHARDED_RECORDER}" ]] || { echo "Error: sharded perf recorder not found: ${SHARDED_RECORDER}" >&2; exit 2; } + +mkdir -p "${control_dir}" "${output_dir}/worker_${worker_id}" + +prefix="${control_dir}/.perf" +pid_file="${control_dir}/.presto_worker_pid_w${worker_id}" +ready_file="${prefix}_sidecar_ready_w${worker_id}" +sidecar_error_file="${prefix}_sidecar_error_w${worker_id}" +shutdown_file="${prefix}_sidecar_shutdown_w${worker_id}" +done_file="${prefix}_sidecar_done_w${worker_id}" +active_perf_pid="" +active_perf_control_ready=0 +active_control_fd="" +active_ack_fd="" +active_control_fifo="" +active_ack_fifo="" +capture_staging_root="" +finalization_failed=0 +last_perf_rc=0 +last_perf_stop_forced=0 +declare -a capture_qids=() +declare -a capture_staging_dirs=() +declare -a capture_session_dirs=() + +rm -f "${ready_file}" "${sidecar_error_file}" "${shutdown_file}" "${done_file}" + +write_sidecar_error() { + printf 'Error: worker %s perf sidecar: %s\n' "${worker_id}" "$*" | tee -a "${sidecar_error_file}" >&2 +} + +log() { + printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" +} + +# perf_event_open consumes a descriptor for each event/target combination. +# Presto creates most of its driver and I/O pools lazily during Q1, so an early +# preflight can need substantially fewer descriptors than a later query. Treat +# --nofile-limit as a minimum, never lower an inherited soft limit, and use all +# descriptor headroom already granted to this Slurm job. Raising RLIMIT_NOFILE +# only raises a ceiling; it does not allocate descriptors by itself. +nofile_soft_before="$(ulimit -Sn)" +nofile_hard="$(ulimit -Hn)" +if [[ "${nofile_hard}" != "unlimited" ]] && (( nofile_hard < nofile_limit )); then + write_sidecar_error "host hard nofile limit ${nofile_hard} is below the requested minimum ${nofile_limit}" + exit 1 +fi +nofile_target="${nofile_hard}" +if [[ "${nofile_soft_before}" != "${nofile_target}" ]] && ! ulimit -Sn "${nofile_target}"; then + write_sidecar_error "cannot raise host perf nofile limit from ${nofile_soft_before} to ${nofile_target} (hard=${nofile_hard})" + exit 1 +fi +nofile_soft_after="$(ulimit -Sn)" +if [[ "${nofile_soft_after}" != "unlimited" ]] && (( nofile_soft_after < nofile_limit )); then + write_sidecar_error "host perf nofile limit ${nofile_soft_after} is below the requested minimum ${nofile_limit} (hard=${nofile_hard})" + exit 1 +fi + +perf_process_is_running() { + local state="" + [[ -n "${active_perf_pid}" ]] || return 1 + kill -0 "${active_perf_pid}" 2>/dev/null || return 1 + state="$(awk '{ print $3 }' "/proc/${active_perf_pid}/stat" 2>/dev/null || true)" + [[ "${state}" != "Z" ]] +} + +close_active_perf_control() { + if [[ "${active_ack_fd}" =~ ^[0-9]+$ ]]; then + exec {active_ack_fd}>&- + fi + if [[ "${active_control_fd}" =~ ^[0-9]+$ ]]; then + exec {active_control_fd}>&- + fi + active_ack_fd="" + active_control_fd="" + [[ -z "${active_ack_fifo}" ]] || rm -f "${active_ack_fifo}" + [[ -z "${active_control_fifo}" ]] || rm -f "${active_control_fifo}" + active_ack_fifo="" + active_control_fifo="" + active_perf_control_ready=0 +} + +send_perf_control() { + local -r command="$1" + local -r timeout_seconds="$2" + local response="" + local tick=0 + + [[ "${active_control_fd}" =~ ^[0-9]+$ && "${active_ack_fd}" =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "${command}" >&"${active_control_fd}" || return 1 + for ((tick = 0; tick < timeout_seconds * 5; ++tick)); do + if IFS= read -r -t 0.2 -u "${active_ack_fd}" response; then + [[ "${response}" == "ack" ]] && return 0 + log "ignoring unexpected perf control response '${response}' for '${command}'" + fi + perf_process_is_running || return 1 + done + return 2 +} + +stop_active_perf() { + last_perf_rc=0 + last_perf_stop_forced=0 + if [[ -n "${active_perf_pid}" ]]; then + local -r stop_marker="${control_dir}/.perf_record_stop_forced_w${worker_id}" + local watchdog_pid="" + rm -f "${stop_marker}" + if perf_process_is_running; then + if [[ "${active_perf_control_ready}" != "1" ]] \ + || ! send_perf_control "stop" "${control_ack_timeout}"; then + kill -INT "${active_perf_pid}" 2>/dev/null || true + fi + ( + sleep "${record_stop_timeout}" + if kill -0 "${active_perf_pid}" 2>/dev/null; then + touch "${stop_marker}" + kill -TERM "${active_perf_pid}" 2>/dev/null || true + sleep 10 + kill -KILL "${active_perf_pid}" 2>/dev/null || true + fi + ) & + watchdog_pid=$! + fi + wait "${active_perf_pid}" 2>/dev/null || last_perf_rc=$? + if [[ -n "${watchdog_pid}" ]]; then + kill -TERM "${watchdog_pid}" 2>/dev/null || true + wait "${watchdog_pid}" 2>/dev/null || true + fi + if [[ -f "${stop_marker}" ]]; then + last_perf_stop_forced=1 + rm -f "${stop_marker}" + fi + fi + active_perf_pid="" + close_active_perf_control +} + +cleanup() { + stop_active_perf + # Every successfully finalized capture is moved out of this directory. Keep + # it (and any partial raw capture) when it is unexpectedly non-empty. + [[ -z "${capture_staging_root}" ]] || rmdir "${capture_staging_root}" 2>/dev/null || true +} +trap cleanup EXIT +trap 'exit 130' INT TERM + +# perf's DWARF records can be write-heavy even at a modest frequency. Retain all +# active-query captures on node-local storage, then atomically publish the +# completed files only after the suite. This keeps NFS/home throughput out of +# both the sampling path and subsequent measured queries. +capture_staging_base="${PERF_CAPTURE_TMPDIR:-${SLURM_TMPDIR:-/tmp}}" +if [[ ! -d "${capture_staging_base}" || ! -w "${capture_staging_base}" ]]; then + write_sidecar_error "perf staging directory is not writable: ${capture_staging_base}" + exit 1 +fi +capture_staging_root="$(mktemp -d "${capture_staging_base%/}/presto-perf-${SLURM_JOB_ID:-nojob}-w${worker_id}.XXXXXX")" || { + write_sidecar_error "could not create node-local perf staging directory below ${capture_staging_base}" + exit 1 +} + +if ! command -v perf >/dev/null 2>&1; then + write_sidecar_error "perf is not installed on host $(hostname). Install the kernel-matched perf package on compute nodes." + exit 1 +fi +perf_record_help="$(perf record -h 2>&1 || true)" +if [[ "${perf_record_help}" != *"--control"* || "${perf_record_help}" != *"--delay"* ]]; then + write_sidecar_error "host perf does not support the --control/--delay handshake required for query-scoped capture" + exit 1 +fi + +# Grace advertises Arm SVE, so perf's automatic DWARF register mask includes +# the extended VG pseudo-register. Hardware core PMUs support it, but the Linux +# cpu-clock/task-clock software PMUs do not advertise EXTENDED_REGS. The kernel +# consequently rejects an otherwise valid software-event recording with +# EOPNOTSUPP. Preserve every base Arm64 GPR needed by normal DWARF CFI while +# omitting only VG. Explicit --user-regs values always win; "perf-default" +# restores perf's own automatic mask. +resolved_user_regs="${user_regs}" +if [[ "${resolved_user_regs}" == "auto" ]]; then + resolved_user_regs="" + if [[ "$(uname -m)" =~ ^(aarch64|arm64)$ \ + && "${call_graph}" == dwarf* \ + && "${event}" =~ ^(cpu-clock|task-clock)(:|$) ]]; then + resolved_user_regs="x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19,x20,x21,x22,x23,x24,x25,x26,x27,x28,x29,lr,sp,pc" + fi +elif [[ "${resolved_user_regs}" == "perf-default" ]]; then + resolved_user_regs="" +fi +declare -a perf_sampling_args=( + -e "${event}" + -F "${frequency}" + -g + --call-graph "${call_graph}" +) +if [[ -n "${resolved_user_regs}" ]]; then + [[ "${perf_record_help}" == *"--user-regs"* ]] || { + write_sidecar_error "host perf does not support --user-regs, required for '${event}' with '${call_graph}' on $(uname -m)" + exit 1 + } + perf_sampling_args+=("--user-regs=${resolved_user_regs}") +fi +perf_require_symbols="${PERF_REQUIRE_SYMBOLS:-1}" +[[ "${perf_require_symbols}" =~ ^[01]$ ]] || { + write_sidecar_error "PERF_REQUIRE_SYMBOLS must be 0 or 1; got ${perf_require_symbols}" + exit 1 +} + +worker_pid="" +for ((waited = 0; waited < pid_wait_timeout; ++waited)); do + if [[ -s "${pid_file}" ]]; then + read -r candidate_pid < "${pid_file}" || true + if [[ "${candidate_pid:-}" =~ ^[0-9]+$ ]] && kill -0 "${candidate_pid}" 2>/dev/null; then + worker_pid="${candidate_pid}" + break + fi + fi + sleep 1 +done + +if [[ -z "${worker_pid}" ]]; then + write_sidecar_error "did not receive a live presto_server PID in ${pid_file} within ${pid_wait_timeout}s" + exit 1 +fi + +# The PID file is written immediately after forking numactl/presto_server. Wait +# for numactl to exec the actual worker so symbol roots and ownership checks are +# against presto_server rather than its short-lived launcher image. +worker_exe="" +for _ in {1..60}; do + worker_exe="$(readlink "/proc/${worker_pid}/exe" 2>/dev/null || true)" + if [[ "$(basename -- "${worker_exe:-unknown}")" == "presto_server" ]]; then + break + fi + sleep 1 +done +if [[ "$(basename -- "${worker_exe:-unknown}")" != "presto_server" ]]; then + write_sidecar_error "PID ${worker_pid} is not presto_server (exe=${worker_exe:-unavailable})" + exit 1 +fi + +worker_uid="$(stat -c '%u' "/proc/${worker_pid}" 2>/dev/null || true)" +if [[ -z "${worker_uid}" || "${worker_uid}" != "$(id -u)" ]]; then + write_sidecar_error "PID ${worker_pid} is owned by uid ${worker_uid:-unknown}, sidecar uid is $(id -u)" + exit 1 +fi + +# Preserve the exact executable mappings behind /proc//root once per run. +# Besides making perf.data portable, a stable ordinary-directory symfs avoids +# symbol lookup failures against Enroot's private mount namespace while the +# worker is live. Anonymous mappings and data-only files cannot contribute +# symbols to a sampled call chain and are intentionally omitted. +profile_root="${output_dir}/worker_${worker_id}" +symbol_root="${profile_root}/symfs" +symbol_manifest="${profile_root}/symbol-manifest.tsv" +mkdir -p "${symbol_root}" +cp "/proc/${worker_pid}/maps" "${profile_root}/worker.maps" +: > "${symbol_manifest}" +while IFS= read -r mapped_file; do + [[ -n "${mapped_file}" ]] || continue + source_file="/proc/${worker_pid}/root${mapped_file}" + destination_file="${symbol_root}${mapped_file}" + [[ -f "${source_file}" && -r "${source_file}" ]] || continue + mkdir -p "$(dirname -- "${destination_file}")" + if cp -L --reflink=auto --preserve=mode,timestamps -- "${source_file}" "${destination_file}"; then + printf '%s\t%s\n' "$(stat -c '%s' "${destination_file}")" "${mapped_file}" >> "${symbol_manifest}" + else + printf 'warning: could not snapshot mapped executable %s\n' "${mapped_file}" >&2 + fi +done < <(awk '$2 ~ /x/ && $6 ~ /^\// { print $6 }' "/proc/${worker_pid}/maps" | sort -u) + +worker_snapshot="${symbol_root}${worker_exe}" +if [[ ! -s "${worker_snapshot}" ]]; then + write_sidecar_error "could not snapshot ${worker_exe} from the worker mount namespace" + exit 1 +fi +if command -v file >/dev/null 2>&1; then + file -L "${worker_snapshot}" > "${profile_root}/worker-executable.txt" 2>&1 || true +fi +if command -v readelf >/dev/null 2>&1; then + readelf -SW "${worker_snapshot}" > "${profile_root}/worker-sections.txt" 2>&1 || true + if grep -qE '[[:space:]]\.(symtab|debug_info)[[:space:]]' "${profile_root}/worker-sections.txt"; then + worker_symbol_table="available" + else + worker_symbol_table="unavailable" + fi +else + worker_symbol_table="unchecked (readelf unavailable)" +fi + +preflight_log="${output_dir}/worker_${worker_id}/preflight.txt" +declare -a preflight_tids=() +for task_dir in "/proc/${worker_pid}/task/"*; do + tid="${task_dir##*/}" + [[ "${tid}" =~ ^[1-9][0-9]*$ ]] && preflight_tids+=("${tid}") +done +mapfile -t preflight_tids < <(printf '%s\n' "${preflight_tids[@]}" | sort -n) +worker_threads_at_preflight="${#preflight_tids[@]}" +if (( ${#preflight_tids[@]} > threads_per_shard )); then + preflight_tids=("${preflight_tids[@]:0:threads_per_shard}") +fi +preflight_tid_csv="$(IFS=,; echo "${preflight_tids[*]}")" +{ + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "hostname=$(hostname)" + echo "worker_id=${worker_id}" + echo "worker_pid=${worker_pid}" + echo "worker_exe=${worker_exe}" + echo "event=${event}" + echo "frequency=${frequency}" + echo "call_graph=${call_graph}" + echo "user_regs_requested=${user_regs}" + echo "user_regs_resolved=${resolved_user_regs:-perf-default}" + echo "record_stop_timeout=${record_stop_timeout}" + echo "control_ack_timeout=${control_ack_timeout}" + echo "startup_attempts=${startup_attempts}" + echo "startup_ready_timeout=${startup_ready_timeout}" + echo "nofile_requested_minimum=${nofile_limit}" + echo "nofile_target=${nofile_target}" + echo "nofile_soft_before=${nofile_soft_before}" + echo "nofile_soft_after=${nofile_soft_after}" + echo "nofile_hard=${nofile_hard}" + echo "worker_threads_at_preflight=${worker_threads_at_preflight}" + echo "threads_per_perf_shard=${threads_per_shard}" + echo "preflight_target_threads=${#preflight_tids[@]}" + echo "symbol_root=${symbol_root}" + echo "worker_symbol_table=${worker_symbol_table}" + echo "capture_staging_root=${capture_staging_root}" + echo "capture_staging_available_kb=$(df -Pk "${capture_staging_root}" | awk 'NR==2 {print $4}')" + echo "postprocess_after_suite=${postprocess}" + echo "perf_event_paranoid=$(cat /proc/sys/kernel/perf_event_paranoid 2>/dev/null || echo unavailable)" + echo "kptr_restrict=$(cat /proc/sys/kernel/kptr_restrict 2>/dev/null || echo unavailable)" + echo "kernel=$(uname -srvmo)" + perf version +} > "${preflight_log}" + +if [[ "${perf_require_symbols}" == "1" && "${worker_symbol_table}" == "unavailable" ]]; then + write_sidecar_error "${worker_exe} has no .symtab or .debug_info section; use a symbol-bearing worker image or set PERF_REQUIRE_SYMBOLS=0 to retain raw address-only captures" + exit 1 +fi + +# Exercise sampling and the requested call-graph/register attributes against +# the actual target before acknowledging readiness. `perf stat` only proves +# that an event can count; it does not detect a PMU rejecting overflow +# interrupts, DWARF registers, or stack sampling. +preflight_data="${capture_staging_root}/preflight.perf.data" +if ! perf record \ + "${perf_sampling_args[@]}" \ + --inherit \ + -t "${preflight_tid_csv}" \ + -o "${preflight_data}" \ + -- sleep 0.2 \ + >"${output_dir}/worker_${worker_id}/preflight-perf-stdout.txt" \ + 2>"${output_dir}/worker_${worker_id}/preflight-perf-stderr.txt"; then + write_sidecar_error "cannot sample '${event}' with call graph '${call_graph}' on PID ${worker_pid}; see profiles/perf/worker_${worker_id}/preflight-perf-stderr.txt" + exit 1 +fi +echo "preflight_perf_data_bytes=$(stat -c '%s' "${preflight_data}")" >> "${preflight_log}" +rm -f "${preflight_data}" + +touch "${ready_file}" +log "perf sidecar ready: worker=${worker_id} pid=${worker_pid} exe=${worker_exe}" + +write_capture_metadata() { + local -r session_dir="$1" + local -r qid="$2" + local -r phase="$3" + { + echo "capture_phase=${phase}" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "hostname=$(hostname)" + echo "slurm_job_id=${SLURM_JOB_ID:-unknown}" + echo "worker_id=${worker_id}" + echo "worker_pid=${worker_pid}" + echo "query_profile_id=${qid}" + echo "event=${event}" + echo "frequency=${frequency}" + echo "call_graph=${call_graph}" + echo "user_regs=${resolved_user_regs:-perf-default}" + echo "symbol_root=${symbol_root}" + echo "worker_symbol_table=${worker_symbol_table}" + echo "worker_exe=$(readlink "/proc/${worker_pid}/exe" 2>/dev/null || echo unavailable)" + } >> "${session_dir}/capture-metadata.txt" +} + +capture_one() { + local -r qid="$1" + local -r session_dir="${output_dir}/worker_${worker_id}/${qid}" + local -r error_file="${prefix}_error_w${worker_id}_${qid}" + local -r started_file="${prefix}_started_w${worker_id}_${qid}" + local -r stop_file="${prefix}_stop_w${worker_id}_${qid}" + local -r finished_file="${prefix}_finished_w${worker_id}_${qid}" + local -r staged_capture_dir="${capture_staging_root}/${qid}" + local worker_threads_at_capture="" + local worker_mappings_at_capture="" + local worker_allowed_cpus="" + local perf_shard_count="" + local perf_total_open_fds="" + local perf_max_shard_open_fds="" + local perf_startup_attempt_count="" + local perf_startup_retry_count="" + local perf_snapshot_tid_count="" + local perf_attached_tid_count="" + local perf_dropped_tid_occurrence_count="" + local perf_dropped_unique_tid_count="" + local expected_shards=0 + local staged_total_bytes=0 + local shard_file="" + local lost_samples="" + local control_rc=0 + local perf_rc=0 + local -a staged_shards=() + local -a stderr_files=() + local -a recorder_args=() + + mkdir -p "${session_dir}" "${staged_capture_dir}" + rm -f \ + "${error_file}" \ + "${started_file}" \ + "${stop_file}" \ + "${finished_file}" + : > "${session_dir}/capture-metadata.txt" + worker_threads_at_capture="$(find "/proc/${worker_pid}/task" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l)" + worker_mappings_at_capture="$(wc -l < "/proc/${worker_pid}/maps" 2>/dev/null || echo unavailable)" + worker_allowed_cpus="$(awk '/^Cpus_allowed_list:/ { print $2 }' "/proc/${worker_pid}/status" 2>/dev/null || true)" + write_capture_metadata "${session_dir}" "${qid}" "start" + { + echo "worker_threads_at_capture_start=${worker_threads_at_capture}" + echo "worker_mappings_at_capture_start=${worker_mappings_at_capture}" + echo "worker_allowed_cpus=${worker_allowed_cpus:-unavailable}" + echo "sidecar_nofile_soft_at_capture=$(ulimit -Sn)" + echo "sidecar_open_fds_before_perf=$(find "/proc/$$/fd" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)" + echo "perf_target_mode=sharded-tids-with-inheritance" + echo "perf_threads_per_shard=${threads_per_shard}" + } >> "${session_dir}/capture-metadata.txt" + cp "/proc/${worker_pid}/maps" "${session_dir}/worker.maps" 2>/dev/null || true + tr '\0' ' ' < "/proc/${worker_pid}/cmdline" > "${session_dir}/worker.cmdline.txt" 2>/dev/null || true + + active_control_fifo="${capture_staging_root}/${qid}.control" + active_ack_fifo="${capture_staging_root}/${qid}.ack" + rm -f "${active_control_fifo}" "${active_ack_fifo}" + if ! mkfifo "${active_control_fifo}" "${active_ack_fifo}"; then + echo "Error: could not create perf control FIFOs for ${qid}" | tee "${error_file}" >&2 + touch "${finished_file}" + return 1 + fi + if ! exec {active_control_fd}<>"${active_control_fifo}"; then + echo "Error: could not open perf control FIFO for ${qid}" | tee "${error_file}" >&2 + close_active_perf_control + touch "${finished_file}" + return 1 + fi + if ! exec {active_ack_fd}<>"${active_ack_fifo}"; then + echo "Error: could not open perf acknowledgement FIFO for ${qid}" | tee "${error_file}" >&2 + close_active_perf_control + touch "${finished_file}" + return 1 + fi + + log "starting perf capture for ${qid}: worker=${worker_id} pid=${worker_pid} threads=${worker_threads_at_capture} nofile=$(ulimit -Sn)" + recorder_args=( + --target-pid "${worker_pid}" + --output-dir "${staged_capture_dir}" + --diagnostic-dir "${session_dir}" + --control-fifo "${active_control_fifo}" + --ack-fifo "${active_ack_fifo}" + --event "${event}" + --frequency "${frequency}" + --call-graph "${call_graph}" + --threads-per-shard "${threads_per_shard}" + --control-ack-timeout "${control_ack_timeout}" + --record-stop-timeout "${record_stop_timeout}" + --startup-attempts "${startup_attempts}" + --startup-ready-timeout "${startup_ready_timeout}" + ) + if [[ -n "${resolved_user_regs}" ]]; then + recorder_args+=(--user-regs "${resolved_user_regs}") + fi + bash "${SHARDED_RECORDER}" "${recorder_args[@]}" \ + >"${session_dir}/perf-record.stdout.txt" \ + 2>"${session_dir}/perf-record.stderr.txt" & + active_perf_pid=$! + + # Every perf shard starts with events disabled. The helper acknowledges only + # after all shards have opened their disjoint TID sets and acknowledged + # enable. Therefore .perf_started remains an actual whole-worker attach + # barrier rather than a timing guess. + send_perf_control "enable" "${control_ack_timeout}" || control_rc=$? + if (( control_rc != 0 )); then + stop_active_perf + perf_rc="${last_perf_rc}" + staged_shards=("${staged_capture_dir}"/perf.data.part*) + staged_total_bytes=0 + for shard_file in "${staged_shards[@]}"; do + [[ -s "${shard_file}" ]] || continue + ((staged_total_bytes += $(stat -c '%s' "${shard_file}"))) + done + write_capture_metadata "${session_dir}" "${qid}" "attach_failed" + { + echo "capture_valid=0" + echo "capture_failure_stage=attach" + echo "capture_failure_control_status=${control_rc}" + echo "capture_failure_perf_status=${perf_rc}" + echo "discarded_partial_perf_data_shards=${#staged_shards[@]}" + echo "discarded_partial_perf_data_bytes=${staged_total_bytes}" + } >> "${session_dir}/capture-metadata.txt" + # The query has not started, so these files contain only an incomplete + # recorder setup—not query samples. Retain the detailed stderr, startup + # attempts, target TIDs, and shard manifest, but do not publish hundreds of + # megabytes of unusable partial perf.data to the shared result directory. + if (( ${#staged_shards[@]} > 0 )); then + rm -f -- "${staged_shards[@]}" + fi + rmdir "${staged_capture_dir}" 2>/dev/null || true + printf 'Error: perf did not acknowledge attachment before %s (control status %s, perf status %s); see %s\n' \ + "${qid}" "${control_rc}" "${perf_rc}" "${session_dir}/perf-record.stderr.txt" | tee "${error_file}" >&2 + touch "${finished_file}" + return 1 + fi + active_perf_control_ready=1 + if [[ -s "${session_dir}/perf-shards.tsv" ]]; then + perf_shard_count="$(awk 'NR > 1 { count++ } END { print count + 0 }' "${session_dir}/perf-shards.tsv")" + perf_attached_tid_count="$(awk 'NR > 1 { total += $4 } END { print total + 0 }' "${session_dir}/perf-shards.tsv")" + perf_total_open_fds="$(awk 'NR > 1 { total += $8 } END { print total + 0 }' "${session_dir}/perf-shards.tsv")" + perf_max_shard_open_fds="$(awk 'NR > 1 && $8 > maximum { maximum = $8 } END { print maximum + 0 }' "${session_dir}/perf-shards.tsv")" + else + perf_shard_count="unavailable" + perf_attached_tid_count="unavailable" + perf_total_open_fds="unavailable" + perf_max_shard_open_fds="unavailable" + fi + if [[ -s "${session_dir}/perf-target-tids.txt" ]]; then + perf_snapshot_tid_count="$(wc -l < "${session_dir}/perf-target-tids.txt")" + else + perf_snapshot_tid_count="unavailable" + fi + if [[ -s "${session_dir}/perf-dropped-tids.tsv" ]]; then + perf_dropped_tid_occurrence_count="$(awk 'NR > 1 { count++ } END { print count + 0 }' "${session_dir}/perf-dropped-tids.tsv")" + perf_dropped_unique_tid_count="$(awk -F '\t' 'NR > 1 && !seen[$3]++ { count++ } END { print count + 0 }' "${session_dir}/perf-dropped-tids.tsv")" + else + perf_dropped_tid_occurrence_count="unavailable" + perf_dropped_unique_tid_count="unavailable" + fi + if [[ -s "${session_dir}/perf-startup-attempts.tsv" ]]; then + perf_startup_attempt_count="$(awk 'NR > 1 { count++ } END { print count + 0 }' "${session_dir}/perf-startup-attempts.tsv")" + perf_startup_retry_count="$(awk -F '\t' 'NR > 1 && $3 == "failed" { count++ } END { print count + 0 }' "${session_dir}/perf-startup-attempts.tsv")" + else + perf_startup_attempt_count="unavailable" + perf_startup_retry_count="unavailable" + fi + { + echo "perf_control_handshake=enabled" + echo "perf_shard_count=${perf_shard_count}" + echo "perf_total_open_fds_after_enable=${perf_total_open_fds}" + echo "perf_max_shard_open_fds_after_enable=${perf_max_shard_open_fds}" + echo "perf_startup_attempt_count=${perf_startup_attempt_count}" + echo "perf_startup_retry_count=${perf_startup_retry_count}" + echo "perf_snapshot_tid_count=${perf_snapshot_tid_count}" + echo "perf_attached_tid_count=${perf_attached_tid_count}" + echo "perf_dropped_tid_occurrence_count=${perf_dropped_tid_occurrence_count}" + echo "perf_dropped_unique_tid_count=${perf_dropped_unique_tid_count}" + } >> "${session_dir}/capture-metadata.txt" + touch "${started_file}" + log "perf attached for ${qid}: helper_pid=${active_perf_pid} shards=${perf_shard_count} tids=${perf_attached_tid_count}/${perf_snapshot_tid_count} dropped=${perf_dropped_unique_tid_count} total_open_fds=${perf_total_open_fds} max_shard_open_fds=${perf_max_shard_open_fds}" + + while [[ ! -f "${stop_file}" && ! -f "${shutdown_file}" ]]; do + if ! kill -0 "${worker_pid}" 2>/dev/null; then + echo "Error: presto_server PID ${worker_pid} exited during ${qid}" | tee "${error_file}" >&2 + break + fi + if ! perf_process_is_running; then + wait "${active_perf_pid}" || perf_rc=$? + active_perf_pid="" + close_active_perf_control + echo "Error: perf record exited during ${qid} with status ${perf_rc}" | tee "${error_file}" >&2 + break + fi + sleep 0.2 + done + rm -f "${stop_file}" + + if [[ -n "${active_perf_pid}" ]]; then + stop_active_perf + perf_rc="${last_perf_rc}" + if [[ "${last_perf_stop_forced}" == "1" ]]; then + echo "Error: perf record did not stop within ${record_stop_timeout}s for ${qid}; it was terminated" \ + | tee "${error_file}" >&2 + fi + fi + # perf versions that do not convert our requested SIGINT into a clean zero + # still finalize perf.data before returning the conventional 128+SIGINT. + [[ "${perf_rc}" == "130" ]] && perf_rc=0 + write_capture_metadata "${session_dir}" "${qid}" "stop" + + if (( perf_rc != 0 )) && [[ ! -s "${error_file}" ]]; then + echo "Error: perf record for ${qid} exited with status ${perf_rc}" | tee "${error_file}" >&2 + fi + staged_shards=("${staged_capture_dir}"/perf.data.part*) + expected_shards="$(awk 'NR > 1 { count++ } END { print count + 0 }' "${session_dir}/perf-shards.tsv" 2>/dev/null || echo 0)" + for shard_file in "${staged_shards[@]}"; do + [[ -s "${shard_file}" ]] || continue + ((staged_total_bytes += $(stat -c '%s' "${shard_file}"))) + done + if (( ${#staged_shards[@]} == 0 || staged_total_bytes == 0 )); then + echo "Error: perf produced no data for ${qid}" | tee "${error_file}" >&2 + else + if (( expected_shards > 0 && ${#staged_shards[@]} != expected_shards )); then + echo "Error: perf produced ${#staged_shards[@]} of ${expected_shards} expected raw shards for ${qid}" \ + | tee "${error_file}" >&2 + fi + capture_qids+=("${qid}") + capture_staging_dirs+=("${staged_capture_dir}") + capture_session_dirs+=("${session_dir}") + { + echo "node_local_perf_data_shards=${#staged_shards[@]}" + echo "node_local_perf_data_bytes=${staged_total_bytes}" + } >> "${session_dir}/capture-metadata.txt" + write_capture_metadata "${session_dir}" "${qid}" "queued" + fi + stderr_files=("${session_dir}/perf-record.stderr.txt" "${session_dir}"/perf-record.part*.stderr.txt) + lost_samples="$(grep -hEo 'lost [0-9]+([.][0-9]+)?%' "${stderr_files[@]}" 2>/dev/null | tail -n 1 || true)" + if [[ -n "${lost_samples}" ]]; then + echo "sample_loss=${lost_samples#lost }" >> "${session_dir}/capture-metadata.txt" + else + echo "sample_loss=0%" >> "${session_dir}/capture-metadata.txt" + fi + if [[ -s "${error_file}" ]]; then + echo "capture_valid=0" >> "${session_dir}/capture-metadata.txt" + else + echo "capture_valid=1" >> "${session_dir}/capture-metadata.txt" + fi + + # This token means perf record has stopped, closed its output, and the raw + # capture is safely retained on node-local storage. Publication and derived + # report generation deliberately happen after every benchmark query. + touch "${finished_file}" + if (( staged_total_bytes > 0 )); then + log "capture stopped for ${qid}; queued ${#staged_shards[@]} shard(s), ${staged_total_bytes} node-local bytes" + fi + [[ ! -s "${error_file}" ]] +} + +publish_capture() { + local -r qid="$1" + local -r staged_capture_dir="$2" + local -r session_dir="$3" + local -r error_file="${prefix}_error_w${worker_id}_${qid}" + local staged_data_file="" + local published_name="" + local published_data_file="" + local publishing_data_file="" + local data_bytes=0 + local total_bytes=0 + local -a staged_shards=() + local -a valid_shards=() + local -a published_names=() + + staged_shards=("${staged_capture_dir}"/perf.data.part*) + for staged_data_file in "${staged_shards[@]}"; do + [[ -s "${staged_data_file}" ]] && valid_shards+=("${staged_data_file}") + done + if (( ${#valid_shards[@]} == 0 )); then + echo "Error: node-local perf data disappeared before publication for ${qid}: ${staged_capture_dir}" \ + | tee -a "${error_file}" >&2 + write_sidecar_error "node-local capture disappeared before publication for ${qid}" + return 1 + fi + + for staged_data_file in "${valid_shards[@]}"; do + ((total_bytes += $(stat -c '%s' "${staged_data_file}"))) + done + log "publishing ${qid}: ${#valid_shards[@]} shard(s), ${total_bytes} bytes from ${staged_capture_dir}" + write_capture_metadata "${session_dir}" "${qid}" "publish_start" + + for staged_data_file in "${valid_shards[@]}"; do + if (( ${#valid_shards[@]} == 1 )); then + published_name="perf.data" + else + published_name="$(basename -- "${staged_data_file}")" + fi + published_data_file="${session_dir}/${published_name}" + publishing_data_file="${session_dir}/.${published_name}.publishing" + data_bytes="$(stat -c '%s' "${staged_data_file}")" + rm -f "${publishing_data_file}" + + # mv performs a copy followed by unlink across filesystems. The + # dot-prefixed destination remains visibly incomplete until the final + # same-filesystem rename; if copying is interrupted, the node-local source + # remains available. + if ! mv -- "${staged_data_file}" "${publishing_data_file}"; then + echo "Error: could not copy node-local perf shard for ${qid} into ${session_dir}" \ + | tee -a "${error_file}" >&2 + write_sidecar_error "could not publish node-local perf data for ${qid}" + return 1 + fi + if ! mv -- "${publishing_data_file}" "${published_data_file}"; then + echo "Error: could not atomically finalize ${published_name} for ${qid} in ${session_dir}" \ + | tee -a "${error_file}" >&2 + write_sidecar_error "could not finalize published perf data for ${qid}" + return 1 + fi + published_names+=("${published_name}") + log "published ${qid} shard: ${published_data_file} (${data_bytes} bytes)" + done + + { + echo "perf_data_shards=${#published_names[@]}" + echo "perf_data_bytes=${total_bytes}" + echo "perf_data_files=$(IFS=,; echo "${published_names[*]}")" + } >> "${session_dir}/capture-metadata.txt" + write_capture_metadata "${session_dir}" "${qid}" "published" + rmdir "${staged_capture_dir}" 2>/dev/null || true + log "published ${qid}: ${#published_names[@]} raw shard(s), ${total_bytes} bytes" +} + +shopt -s nullglob +while [[ ! -f "${shutdown_file}" ]]; do + start_tokens=("${prefix}_start_w${worker_id}_"*) + if (( ${#start_tokens[@]} == 0 )); then + sleep 0.2 + continue + fi + + start_token="${start_tokens[0]}" + qid="${start_token#${prefix}_start_w${worker_id}_}" + rm -f "${start_token}" + if [[ ! "${qid}" =~ ^[A-Za-z0-9_.-]+$ ]]; then + write_sidecar_error "unsafe query profile identifier: ${qid}" + break + fi + capture_one "${qid}" || true +done + +rm -f "${ready_file}" +log "benchmark capture phase complete; publishing ${#capture_qids[@]} queued capture(s)" +for ((capture_index = 0; capture_index < ${#capture_qids[@]}; ++capture_index)); do + publish_capture \ + "${capture_qids[capture_index]}" \ + "${capture_staging_dirs[capture_index]}" \ + "${capture_session_dirs[capture_index]}" || finalization_failed=1 +done + +if [[ "${postprocess}" == "1" && "${finalization_failed}" == "0" && ${#capture_qids[@]} -gt 0 ]]; then + log "starting deferred report and flamegraph generation" + if ! bash "${postprocess_script}" --worker-root "${profile_root}" --worker-id "${worker_id}"; then + write_sidecar_error "one or more published captures failed deferred post-processing" + finalization_failed=1 + fi +fi + +rm -f "${shutdown_file}" +touch "${done_file}" +log "perf sidecar complete for worker ${worker_id}; finalization_failed=${finalization_failed}" +exit "${finalization_failed}" diff --git a/presto/slurm/presto-nvl72/perf_profiler_functions.sh b/presto/slurm/presto-nvl72/perf_profiler_functions.sh new file mode 100755 index 00000000..1fbc4d2e --- /dev/null +++ b/presto/slurm/presto-nvl72/perf_profiler_functions.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Query-side half of the host-perf handshake. This file is sourced by the +# benchmark CLI container. The same host LOGS directory is bind-mounted at +# /var/log/nsys, allowing the CLI and the host-side sampler to exchange tokens +# without network calls or PID-namespace assumptions in the CLI container. + +set -e + +_perf_wait_for_token() { + local -r wanted="$1" + local -r error_file="$2" + local -r sidecar_error_file="$3" + local -r timeout="$4" + local -r deadline=$((SECONDS + timeout)) + + while [[ ! -f "${wanted}" ]]; do + if [[ -s "${error_file}" ]]; then + cat "${error_file}" >&2 + return 1 + fi + if [[ -s "${sidecar_error_file}" ]]; then + cat "${sidecar_error_file}" >&2 + return 1 + fi + # These tokens bracket the measured query. A one-second poll interval adds + # a visible idle tail to short CPU profiles, so poll the shared filesystem + # promptly while retaining a wall-clock timeout. + sleep 0.1 + if (( SECONDS >= deadline )); then + echo "Error: timed out after ${timeout}s waiting for $(basename "${wanted}")" >&2 + return 1 + fi + done +} + +start_profiler() { + local -r qid="$(basename -- "$1")" + local -r worker_id="${PERF_WORKER_ID:-0}" + local -r token_dir="${PERF_TOKEN_DIR:-/var/log/nsys}" + local -r timeout="${PERF_START_TIMEOUT:-120}" + local -r prefix="${token_dir}/.perf" + local -r error_file="${prefix}_error_w${worker_id}_${qid}" + local -r sidecar_error_file="${prefix}_sidecar_error_w${worker_id}" + + [[ "${qid}" =~ ^[A-Za-z0-9_.-]+$ ]] || { + echo "Error: unsafe perf profile identifier: ${qid}" >&2 + return 1 + } + + rm -f \ + "${prefix}_started_w${worker_id}_${qid}" \ + "${prefix}_stop_w${worker_id}_${qid}" \ + "${prefix}_finished_w${worker_id}_${qid}" \ + "${error_file}" + touch "${prefix}_start_w${worker_id}_${qid}" + + _perf_wait_for_token \ + "${prefix}_started_w${worker_id}_${qid}" \ + "${error_file}" \ + "${sidecar_error_file}" \ + "${timeout}" + rm -f "${prefix}_started_w${worker_id}_${qid}" +} + +stop_profiler() { + local -r qid="$(basename -- "$1")" + local -r worker_id="${PERF_WORKER_ID:-0}" + local -r token_dir="${PERF_TOKEN_DIR:-/var/log/nsys}" + # The finished token acknowledges that perf record has stopped and its closed + # raw file is queued safely on node-local storage. Shared-filesystem + # publication and report generation are not part of this query-side timeout. + local -r timeout="${PERF_STOP_TIMEOUT:-600}" + local -r prefix="${token_dir}/.perf" + local -r error_file="${prefix}_error_w${worker_id}_${qid}" + local -r sidecar_error_file="${prefix}_sidecar_error_w${worker_id}" + + touch "${prefix}_stop_w${worker_id}_${qid}" + _perf_wait_for_token \ + "${prefix}_finished_w${worker_id}_${qid}" \ + "${error_file}" \ + "${sidecar_error_file}" \ + "${timeout}" + rm -f "${prefix}_finished_w${worker_id}_${qid}" + + if [[ -s "${error_file}" ]]; then + cat "${error_file}" >&2 + return 1 + fi +} diff --git a/presto/slurm/presto-nvl72/presto-repl-from-job.sh b/presto/slurm/presto-nvl72/presto-repl-from-job.sh new file mode 100755 index 00000000..166a59a8 --- /dev/null +++ b/presto/slurm/presto-nvl72/presto-repl-from-job.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Open presto-cli against the coordinator node of a running Slurm job. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}" + +source ./defaults.env +source ./launcher_common.sh + +VARIANT_TYPE="${VARIANT_TYPE:-${CLUSTER_DEFAULT_VARIANT:-gpu}}" +resolve_cluster_variant "${VARIANT_TYPE}" >/dev/null 2>&1 || true + +IMAGE="${IMAGE:-${COORD_IMAGE:-}}" +PORT="${PORT:-${CLUSTER_DEFAULT_PORT:-9200}}" +CATALOG="${CATALOG:-hive}" +SCHEMA="${SCHEMA:-}" +PRESTO_USER="${PRESTO_USER:-${USER:-prestouser}}" + +JOB_ID="" +SQL_FILE="" +EXTRA_ARGS=() + +usage() { + cat < [-- extra presto-cli args] + +Options: + -s, --schema Schema to pass to presto-cli + -f, --file Execute a SQL file instead of opening an interactive REPL + -i, --image Coordinator/CLI container image + Accepts an absolute path, a .sqsh basename, or an image name + resolved under IMAGE_DIR from cluster_config.env + -p, --port Presto coordinator HTTP port (default: ${PORT}) + -c, --catalog Catalog (default: ${CATALOG}) + -u, --user Presto user (default: ${PRESTO_USER}) + -h, --help Show this help + +Examples: + $0 -s tpchsf10000 5236 + $0 -s tpchsf10000 5243 -f analyze_tables.sql + IMAGE=/path/to/images/coordinator.sqsh $0 5236 +EOF +} + +require_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "${value}" || "${value}" == -* ]]; then + echo "Error: ${flag} requires a value" >&2 + exit 2 + fi +} + +resolve_image_path() { + local image="$1" + if [[ -z "${image}" ]]; then + echo "" + elif [[ "${image}" == /* ]]; then + echo "${image}" + elif [[ "${image}" == *.sqsh ]]; then + if [[ -z "${IMAGE_DIR:-}" ]]; then + echo "Error: IMAGE_DIR is not set; pass an absolute image path with -i" >&2 + exit 2 + fi + echo "${IMAGE_DIR}/${image}" + else + if [[ -z "${IMAGE_DIR:-}" ]]; then + echo "Error: IMAGE_DIR is not set; pass an absolute image path with -i" >&2 + exit 2 + fi + echo "${IMAGE_DIR}/${image}.sqsh" + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -s|--schema) + require_value "$1" "${2:-}" + SCHEMA="$2" + shift 2 + ;; + -f|--file) + require_value "$1" "${2:-}" + SQL_FILE="$2" + shift 2 + ;; + -i|--image) + require_value "$1" "${2:-}" + IMAGE="$2" + shift 2 + ;; + -p|--port) + require_value "$1" "${2:-}" + PORT="$2" + shift 2 + ;; + -c|--catalog) + require_value "$1" "${2:-}" + CATALOG="$2" + shift 2 + ;; + -u|--user) + require_value "$1" "${2:-}" + PRESTO_USER="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + EXTRA_ARGS+=("$@") + break + ;; + -*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + if [[ -z "${JOB_ID}" ]]; then + JOB_ID="$1" + else + EXTRA_ARGS+=("$1") + fi + shift + ;; + esac +done + +if [[ -z "${JOB_ID}" ]]; then + usage >&2 + exit 2 +fi + +IMAGE="$(resolve_image_path "${IMAGE}")" +if [[ -z "${IMAGE}" ]]; then + echo "Error: coordinator image is not set. Pass -i, set IMAGE, or configure CLUSTER_${VARIANT_TYPE^^}_DEFAULT_COORD_IMAGE." >&2 + exit 2 +fi + +MOUNT_ARGS=() +if [[ -n "${SQL_FILE}" ]]; then + if [[ "${SQL_FILE}" != /* ]]; then + SQL_FILE="${PWD}/${SQL_FILE}" + fi + SQL_DIR="$(cd "$(dirname "${SQL_FILE}")" && pwd -P)" + SQL_FILE="${SQL_DIR}/$(basename "${SQL_FILE}")" + if [[ ! -r "${SQL_FILE}" ]]; then + echo "Error: SQL file not readable: ${SQL_FILE}" >&2 + exit 1 + fi + EXTRA_ARGS=(--file "${SQL_FILE}" "${EXTRA_ARGS[@]}") + MOUNT_ARGS=(--container-mounts="${SQL_DIR}:${SQL_DIR}") +fi + +NODELIST="$(squeue -j "${JOB_ID}" -h -o '%N' | awk 'NR==1{print $1}')" +if [[ -z "${NODELIST}" || "${NODELIST}" == "(null)" ]]; then + echo "Could not find allocated nodes for job ${JOB_ID}. Is it running?" >&2 + exit 1 +fi + +COORD="$(scontrol show hostnames "${NODELIST}" | head -1)" +if [[ -z "${COORD}" ]]; then + echo "Could not resolve coordinator node from node list: ${NODELIST}" >&2 + exit 1 +fi + +SERVER="http://${COORD}:${PORT}" + +echo "Job: ${JOB_ID}" +echo "Coordinator: ${COORD}" +echo "Server: ${SERVER}" +echo "Image: ${IMAGE}" +echo "Catalog: ${CATALOG}" +[[ -n "${SCHEMA}" ]] && echo "Schema: ${SCHEMA}" +[[ -n "${SQL_FILE}" ]] && echo "SQL file: ${SQL_FILE}" +echo + +SRUN_TTY_ARGS=() +if [[ -z "${SQL_FILE}" ]]; then + SRUN_TTY_ARGS=(--pty) +fi + +srun --jobid "${JOB_ID}" -w "${COORD}" --ntasks=1 --overlap \ + "${SRUN_TTY_ARGS[@]}" \ + --container-image="${IMAGE}" \ + "${MOUNT_ARGS[@]}" \ + --export=ALL,PRESTO_SERVER="${SERVER}",PRESTO_CATALOG="${CATALOG}",PRESTO_SCHEMA="${SCHEMA}",PRESTO_USER_NAME="${PRESTO_USER}" \ + -- bash -lc ' + set -euo pipefail + export PAGER="${PAGER:-cat}" + export PRESTO_PAGER="${PRESTO_PAGER:-cat}" + args=(presto-cli --server "$PRESTO_SERVER" --catalog "$PRESTO_CATALOG" --user "$PRESTO_USER_NAME") + if [[ -n "${PRESTO_SCHEMA:-}" ]]; then + args+=(--schema "$PRESTO_SCHEMA") + fi + exec "${args[@]}" "$@" + ' presto-cli "${EXTRA_ARGS[@]}" diff --git a/presto/slurm/presto-nvl72/process-perf-captures.sh b/presto/slurm/presto-nvl72/process-perf-captures.sh new file mode 100755 index 00000000..9e30bc71 --- /dev/null +++ b/presto/slurm/presto-nvl72/process-perf-captures.sh @@ -0,0 +1,776 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Generate reports from raw host-perf captures after a benchmark. When invoked +# outside Slurm, this script starts one CPU-variant compute allocation so that +# perf unwinds the capture on the same architecture that recorded it. Raw data +# and the portable symfs are staged onto that node's local disk; only derived +# artifacts are copied back to the shared result directory. + +set -uo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd -P)" +SCRIPT_PATH="${SCRIPT_DIR}/$(basename -- "${BASH_SOURCE[0]}")" +FLAMEGRAPH_SCRIPT="${SCRIPT_DIR}/../../scripts/perf_flamegraph.py" + +worker_id=0 +query_id="" +force=0 +no_inline=0 +local_only=0 +requested_jobs="${PERF_POSTPROCESS_JOBS:-0}" +compute_time="${PERF_POSTPROCESS_TIME:-00:30:00}" +result_dir="" +worker_root="" + +usage() { + cat <. + +Outside an existing Slurm allocation, the default is to submit one blocking +CPU compute-node step with srun. The capture and symfs are copied to node-local +storage, shards are symbolized in parallel, and derived files are atomically +published back into RESULT_DIR. + +Options: + --worker-id N Worker ID below RESULT_DIR/profiles/perf (default: 0) + --worker-root DIR Process this profiles/perf/worker_ directory directly + --query ID Process one capture directory, e.g. Q18_iter1 + --no-inline Disable inline-function expansion in perf report/script + --jobs N Maximum parallel perf processes (default: raw shard count) + --time LIMIT Compute allocation time limit (default: ${compute_time}) + --local Do not launch srun; process on this host/allocation + --force Reprocess a matching completed capture + -h, --help Show this help + +Environment: + CLUSTER_CONFIG Cluster config path (default: + ~/.cluster_config.env) + PERF_POSTPROCESS_PARTITION Override CLUSTER_CPU_PARTITION + PERF_POSTPROCESS_ACCOUNT Override CLUSTER_CPU_ACCOUNT + PERF_POSTPROCESS_TMPDIR Node-local staging base (default: + \${SLURM_TMPDIR:-/tmp}) + PERF_POSTPROCESS_KEEP_TMP=1 Retain successful staging directories +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --worker-id) + worker_id="${2:?missing --worker-id value}" + shift 2 + ;; + --worker-root) + worker_root="${2:?missing --worker-root value}" + shift 2 + ;; + --query) + query_id="${2:?missing --query value}" + shift 2 + ;; + --no-inline) + no_inline=1 + shift + ;; + --jobs) + requested_jobs="${2:?missing --jobs value}" + shift 2 + ;; + --time) + compute_time="${2:?missing --time value}" + shift 2 + ;; + --local) + local_only=1 + shift + ;; + --force) + force=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + -*) + echo "Error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + *) + if [[ -n "${result_dir}" ]]; then + echo "Error: only one RESULT_DIR may be specified" >&2 + usage >&2 + exit 2 + fi + result_dir="$1" + shift + ;; + esac +done + +[[ "${worker_id}" =~ ^[0-9]+$ ]] || { + echo "Error: invalid worker ID: ${worker_id}" >&2 + exit 2 +} +[[ "${requested_jobs}" =~ ^[0-9]+$ ]] || { + echo "Error: --jobs must be a positive integer, or 0 for automatic; got ${requested_jobs}" >&2 + exit 2 +} +[[ "${compute_time}" != *[[:space:]]* && -n "${compute_time}" ]] || { + echo "Error: invalid --time value: ${compute_time}" >&2 + exit 2 +} +if [[ -n "${query_id}" && ! "${query_id}" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "Error: unsafe query profile identifier: ${query_id}" >&2 + exit 2 +fi +if [[ -n "${worker_root}" && -n "${result_dir}" ]]; then + echo "Error: specify RESULT_DIR or --worker-root, not both" >&2 + exit 2 +fi +if [[ -z "${worker_root}" ]]; then + [[ -n "${result_dir}" ]] || { + echo "Error: RESULT_DIR or --worker-root is required" >&2 + usage >&2 + exit 2 + } + worker_root="${result_dir%/}/profiles/perf/worker_${worker_id}" +fi + +[[ -d "${worker_root}" ]] || { + echo "Error: perf worker directory not found: ${worker_root}" >&2 + exit 1 +} +worker_root="$(cd -- "${worker_root}" >/dev/null 2>&1 && pwd -P)" +[[ -d "${worker_root}/symfs" ]] || { + echo "Error: symbol filesystem not found: ${worker_root}/symfs" >&2 + exit 1 +} +[[ -f "${FLAMEGRAPH_SCRIPT}" ]] || { + echo "Error: bundled flamegraph renderer not found: ${FLAMEGRAPH_SCRIPT}" >&2 + exit 1 +} + +shopt -s nullglob + +log() { + printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" +} + +# Populate capture_data_files for one capture directory. +capture_data_files=() +find_capture_data_files() { + local -r session_dir="$1" + local data_file="" + capture_data_files=() + if [[ -s "${session_dir}/perf.data" ]]; then + capture_data_files+=("${session_dir}/perf.data") + fi + for data_file in "${session_dir}"/perf.data.part*; do + [[ -s "${data_file}" ]] && capture_data_files+=("${data_file}") + done +} + +capture_validation_error="" +validate_capture() { + local -r session_dir="$1" + local -r metadata="${session_dir}/capture-metadata.txt" + local -r manifest="${session_dir}/perf-shards.tsv" + local expected_shards="" + local explicit_valid="" + + capture_validation_error="" + if [[ ! -s "${metadata}" ]]; then + capture_validation_error="capture metadata is missing" + return 1 + fi + explicit_valid="$(awk -F= '$1 == "capture_valid" { value=$2 } END { print value }' "${metadata}")" + if [[ "${explicit_valid}" == "0" ]]; then + capture_validation_error="capture metadata marks the recording invalid" + return 1 + fi + if ! grep -qx 'perf_control_handshake=enabled' "${metadata}"; then + capture_validation_error="not every perf shard completed the enable handshake" + return 1 + fi + if ! grep -qx 'capture_phase=queued' "${metadata}"; then + capture_validation_error="recorders did not stop and queue a complete raw capture" + return 1 + fi + if ! grep -qx 'capture_phase=published' "${metadata}"; then + capture_validation_error="raw capture publication did not complete" + return 1 + fi + if [[ ! -s "${manifest}" ]]; then + capture_validation_error="perf shard manifest is missing" + return 1 + fi + if ! awk -F '\t' ' + NR > 1 { + count++ + if ($2 != "stopped" || ($9 != "0" && $9 != "130")) { + invalid=1 + } + } + END { exit !(count > 0 && !invalid) } + ' "${manifest}"; then + capture_validation_error="one or more perf shards did not stop cleanly" + return 1 + fi + + find_capture_data_files "${session_dir}" + expected_shards="$(awk -F= '$1 == "perf_data_shards" { value=$2 } END { print value }' "${metadata}")" + if [[ ! "${expected_shards}" =~ ^[1-9][0-9]*$ ]]; then + capture_validation_error="published shard count is missing from capture metadata" + return 1 + fi + if (( ${#capture_data_files[@]} != expected_shards )); then + capture_validation_error="found ${#capture_data_files[@]} raw shard(s), expected ${expected_shards}" + return 1 + fi +} + +session_dirs=() +max_shards=0 +invalid_capture_count=0 +if [[ -n "${query_id}" ]]; then + [[ -d "${worker_root}/${query_id}" ]] || { + echo "Error: perf capture directory not found: ${worker_root}/${query_id}" >&2 + exit 1 + } + if ! validate_capture "${worker_root}/${query_id}"; then + echo "Error: ${query_id} is not a complete perf capture: ${capture_validation_error}" >&2 + exit 1 + fi + session_dirs=("${worker_root}/${query_id}") + max_shards="${#capture_data_files[@]}" +else + for candidate in "${worker_root}"/*; do + [[ -d "${candidate}" ]] || continue + candidate_name="$(basename -- "${candidate}")" + [[ "${candidate_name}" =~ ^[A-Za-z0-9_.-]+$ ]] || { + log "Skipping unsafe capture directory name: ${candidate_name}" + continue + } + [[ -s "${candidate}/capture-metadata.txt" ]] || continue + if ! validate_capture "${candidate}"; then + log "ERROR ${candidate_name}: refusing incomplete perf capture (${capture_validation_error})" + ((invalid_capture_count += 1)) + continue + fi + session_dirs+=("${candidate}") + if (( ${#capture_data_files[@]} > max_shards )); then + max_shards="${#capture_data_files[@]}" + fi + done +fi + +if (( ${#session_dirs[@]} == 0 )); then + echo "Error: no perf.data captures or shards found below ${worker_root}" >&2 + exit 1 +fi + +parallel_jobs="${max_shards}" +if (( requested_jobs > 0 && requested_jobs < parallel_jobs )); then + parallel_jobs="${requested_jobs}" +fi +(( parallel_jobs > 0 )) || parallel_jobs=1 + +load_cluster_config() { + local cluster_config="${CLUSTER_CONFIG:-${HOME}/.cluster_config.env}" + if [[ -f "${cluster_config}" ]]; then + # shellcheck disable=SC1090 + source "${cluster_config}" + else + log "WARNING cluster config not found at ${cluster_config}; srun will use Slurm defaults" + fi +} + +launch_compute_postprocessor() { + local cluster_cpu_limit="" + local partition="" + local account="" + local -a srun_args=() + local -a forwarded_args=() + + command -v srun >/dev/null 2>&1 || { + echo "Error: srun is not installed; run on a compatible compute node with --local" >&2 + return 1 + } + + load_cluster_config + cluster_cpu_limit="${CLUSTER_CPU_CPUS_PER_TASK:-}" + if [[ "${cluster_cpu_limit}" =~ ^[1-9][0-9]*$ ]] && (( parallel_jobs > cluster_cpu_limit )); then + log "Capping --jobs from ${parallel_jobs} to configured CPU allocation ${cluster_cpu_limit}" + parallel_jobs="${cluster_cpu_limit}" + fi + partition="${PERF_POSTPROCESS_PARTITION:-${CLUSTER_CPU_PARTITION:-}}" + account="${PERF_POSTPROCESS_ACCOUNT:-${CLUSTER_CPU_ACCOUNT:-}}" + + srun_args=( + --nodes=1 + --ntasks=1 + --cpus-per-task="${parallel_jobs}" + --time="${compute_time}" + --job-name=presto-perf-postprocess + --kill-on-bad-exit=1 + --export=ALL + ) + [[ -n "${partition}" ]] && srun_args+=(--partition="${partition}") + [[ -n "${account}" ]] && srun_args+=(--account="${account}") + + forwarded_args=( + --worker-root "${worker_root}" + --worker-id "${worker_id}" + --jobs "${parallel_jobs}" + --time "${compute_time}" + ) + [[ -n "${query_id}" ]] && forwarded_args+=(--query "${query_id}") + (( no_inline )) && forwarded_args+=(--no-inline) + (( force )) && forwarded_args+=(--force) + + log "Launching Arm-compatible compute postprocessor (${parallel_jobs} parallel perf process(es), time ${compute_time})" + ( + export PRESTO_PERF_POSTPROCESS_COMPUTE=1 + exec srun "${srun_args[@]}" bash "${SCRIPT_PATH}" "${forwarded_args[@]}" + ) +} + +# The sidecar already calls this script inside the benchmark allocation. +# PRESTO_PERF_POSTPROCESS_COMPUTE also prevents recursion in lightweight srun +# wrappers that do not expose SLURM_JOB_ID to the launched shell. +if (( ! local_only )) \ + && [[ -z "${SLURM_JOB_ID:-}" ]] \ + && [[ "${PRESTO_PERF_POSTPROCESS_COMPUTE:-0}" != "1" ]]; then + launch_compute_postprocessor + exit $? +fi + +for required_command in perf python3 gzip file mktemp cp sort; do + command -v "${required_command}" >/dev/null 2>&1 || { + echo "Error: ${required_command} is not installed on $(hostname)" >&2 + exit 1 + } +done + +normalize_arch() { + case "$1" in + aarch64|arm64) printf 'aarch64\n' ;; + x86_64|amd64) printf 'x86_64\n' ;; + *) printf '%s\n' "$1" ;; + esac +} + +captured_worker="${worker_root}/symfs/usr/bin/presto_server" +if [[ ! -f "${captured_worker}" ]]; then + captured_worker="$(find "${worker_root}/symfs" -type f -name presto_server -print -quit 2>/dev/null || true)" +fi +if [[ -n "${captured_worker}" ]]; then + captured_file_description="$(file -L "${captured_worker}" 2>/dev/null || true)" + case "${captured_file_description}" in + *aarch64*|*"ARM aarch64"*) captured_arch="aarch64" ;; + *x86-64*|*x86_64*) captured_arch="x86_64" ;; + *) captured_arch="" ;; + esac + current_arch="$(normalize_arch "$(uname -m)")" + if [[ -n "${captured_arch}" && "${captured_arch}" != "${current_arch}" ]]; then + echo "Error: capture contains a ${captured_arch} worker, but $(hostname) is ${current_arch}." >&2 + echo " Omit --local so the script can launch on the CPU compute partition." >&2 + exit 1 + fi +else + captured_arch="" + current_arch="$(normalize_arch "$(uname -m)")" + log "WARNING could not locate presto_server in symfs for an architecture preflight" +fi + +# perf can open one DSO descriptor per mapped image during symbolization. Use +# all descriptor headroom granted to the postprocessing allocation. +nofile_soft_before="$(ulimit -Sn)" +nofile_hard="$(ulimit -Hn)" +if [[ "${nofile_soft_before}" != "${nofile_hard}" ]]; then + ulimit -Sn "${nofile_hard}" 2>/dev/null || true +fi + +stage_base="${PERF_POSTPROCESS_TMPDIR:-${SLURM_TMPDIR:-/tmp}}" +mkdir -p "${stage_base}" || { + echo "Error: cannot create node-local staging base: ${stage_base}" >&2 + exit 1 +} +stage_base="$(cd -- "${stage_base}" >/dev/null 2>&1 && pwd -P)" +stage_root="$(mktemp -d "${stage_base}/presto-perf-postprocess.${SLURM_JOB_ID:-manual}.XXXXXX")" || { + echo "Error: cannot create a node-local staging directory below ${stage_base}" >&2 + exit 1 +} + +cleanup_stage() { + local rc=$? + trap - EXIT + if [[ -n "${stage_root:-}" && -d "${stage_root}" ]]; then + if [[ "${PERF_POSTPROCESS_KEEP_TMP:-0}" == "1" ]]; then + log "Retaining node-local staging directory: ${stage_root}" + else + case "${stage_root}" in + "${stage_base}"/presto-perf-postprocess.*) + rm -rf -- "${stage_root}" + ;; + *) + log "WARNING refusing to remove unexpected staging path: ${stage_root}" + ;; + esac + fi + fi + exit "${rc}" +} +trap cleanup_stage EXIT + +copy_without_metadata() { + cp --no-preserve=mode,ownership,timestamps -- "$1" "$2" +} + +stage_worker="${stage_root}/worker_${worker_id}" +stage_symfs="${stage_worker}/symfs" +mkdir -p "${stage_symfs}" +log "Staging symfs on $(hostname): ${stage_root}" +cp -R --reflink=auto --no-preserve=mode,ownership,timestamps \ + -- "${worker_root}/symfs/." "${stage_symfs}/" || { + echo "Error: could not stage ${worker_root}/symfs" >&2 + exit 1 +} + +perf_inline_args=() +inline_expansion="enabled" +if (( no_inline )); then + perf_inline_args+=(--no-inline) + inline_expansion="disabled" +fi + +batch_pids=() +parallel_failed=0 +wait_for_batch() { + local pid="" + for pid in "${batch_pids[@]}"; do + if ! wait "${pid}"; then + parallel_failed=1 + fi + done + batch_pids=() +} + +discard_staged_session() { + local -r staged_session="$1" + [[ "${PERF_POSTPROCESS_KEEP_TMP:-0}" != "1" ]] || return 0 + case "${staged_session}" in + "${stage_worker}"/*) + rm -rf -- "${staged_session}" + ;; + *) + log "WARNING refusing to remove unexpected staged capture path: ${staged_session}" + ;; + esac +} + +publish_derived_files() { + local -r staged_session="$1" + local -r destination_session="$2" + local -r successful="$3" + local filename="" + local pending="" + local publishing_file="" + local -a files_to_publish=() + local -a derived_names=( + build-ids.txt + build-ids.stderr.txt + perf-report.txt + perf-report.stderr.txt + perf-script.stderr.txt + perf.script.gz + stacks.folded + flamegraph.html + flamegraph-generator.txt + flamegraph-unavailable.txt + postprocess-metadata.txt + ) + + # Copy every new artifact under a hidden temporary name before changing the + # published set. A shared-filesystem copy failure therefore leaves the + # previous complete result intact. Raw perf.data and capture metadata are + # never touched. + for filename in "${derived_names[@]}"; do + [[ -f "${staged_session}/${filename}" ]] || continue + publishing_file="${destination_session}/.${filename}.publishing.${SLURM_JOB_ID:-$$}" + rm -f "${publishing_file}" + if ! copy_without_metadata "${staged_session}/${filename}" "${publishing_file}"; then + echo "Error: could not publish ${filename} to ${destination_session}" >&2 + rm -f "${publishing_file}" + for pending in "${files_to_publish[@]}"; do + rm -f "${destination_session}/.${pending}.publishing.${SLURM_JOB_ID:-$$}" + done + return 1 + fi + files_to_publish+=("${filename}") + done + + rm -f \ + "${destination_session}/.postprocess-complete" \ + "${destination_session}/.postprocess-failed" + for filename in "${derived_names[@]}"; do + if [[ ! -f "${staged_session}/${filename}" ]]; then + rm -f "${destination_session}/${filename}" + fi + done + for filename in "${files_to_publish[@]}"; do + publishing_file="${destination_session}/.${filename}.publishing.${SLURM_JOB_ID:-$$}" + if ! mv -f -- "${publishing_file}" "${destination_session}/${filename}"; then + for pending in "${files_to_publish[@]}"; do + rm -f "${destination_session}/.${pending}.publishing.${SLURM_JOB_ID:-$$}" + done + return 1 + fi + done + if [[ "${successful}" == "1" ]]; then + touch "${destination_session}/.postprocess-complete" + else + touch "${destination_session}/.postprocess-failed" + fi +} + +process_capture() { + local -r source_session="$1" + local -r qid="$(basename -- "${source_session}")" + local -r staged_session="${stage_worker}/${qid}" + local -r script_file="${staged_session}/perf.script" + local -r generator_log="${staged_session}/flamegraph-generator.txt" + local data_file="" + local part_name="" + local part_file="" + local total_data_bytes=0 + local index=0 + local report_failed=0 + local renderer_status="failed" + local existing_inline="" + local -a data_files=() + local -a staged_data_files=() + local -a part_files=() + + if [[ -f "${source_session}/.postprocess-complete" && "${force}" != "1" ]]; then + existing_inline="$(awk -F= '$1 == "postprocess_inline_expansion" { print $2 }' \ + "${source_session}/postprocess-metadata.txt" 2>/dev/null | tail -1)" + if [[ -s "${source_session}/flamegraph.html" \ + && -s "${source_session}/perf.script.gz" \ + && "${existing_inline}" == "${inline_expansion}" ]]; then + log "Skipping ${qid}; already post-processed with inline expansion ${inline_expansion} (use --force to regenerate)" + return 0 + fi + log "Reprocessing ${qid}; existing marker is incomplete or used a different inline mode" + fi + + find_capture_data_files "${source_session}" + data_files=("${capture_data_files[@]}") + if (( ${#data_files[@]} == 0 )); then + log "ERROR ${qid}: missing or empty perf.data capture" + return 1 + fi + + mkdir -p "${staged_session}" + for data_file in "${data_files[@]}"; do + ((total_data_bytes += $(stat -c '%s' "${data_file}"))) + copy_without_metadata "${data_file}" "${staged_session}/$(basename -- "${data_file}")" || { + log "ERROR ${qid}: could not stage $(basename -- "${data_file}")" + return 1 + } + staged_data_files+=("${staged_session}/$(basename -- "${data_file}")") + done + + log "Processing ${qid} (${#staged_data_files[@]} raw shard(s), ${total_data_bytes} bytes, jobs=${parallel_jobs}, inline=${inline_expansion})" + + log "${qid}: build IDs" + parallel_failed=0 + batch_pids=() + index=0 + for data_file in "${staged_data_files[@]}"; do + printf -v part_name 'part%03d' "${index}" + ( + perf buildid-list -i "${data_file}" \ + > "${staged_session}/.build-ids.${part_name}.txt" \ + 2> "${staged_session}/.build-ids.${part_name}.stderr.txt" + ) & + batch_pids+=("$!") + ((index += 1)) + if (( ${#batch_pids[@]} >= parallel_jobs )); then + wait_for_batch + fi + done + wait_for_batch + : > "${staged_session}/build-ids.txt" + : > "${staged_session}/build-ids.stderr.txt" + for part_file in "${staged_session}"/.build-ids.part*.txt; do + cat "${part_file}" >> "${staged_session}/build-ids.txt" + done + for part_file in "${staged_session}"/.build-ids.part*.stderr.txt; do + cat "${part_file}" >> "${staged_session}/build-ids.stderr.txt" + done + sort -u -o "${staged_session}/build-ids.txt" "${staged_session}/build-ids.txt" + rm -f "${staged_session}"/.build-ids.part*.txt "${staged_session}"/.build-ids.part*.stderr.txt + + log "${qid}: parallel text reports" + parallel_failed=0 + batch_pids=() + index=0 + for data_file in "${staged_data_files[@]}"; do + printf -v part_name 'part%03d' "${index}" + ( + perf report "${perf_inline_args[@]}" \ + --stdio --no-children --percent-limit 0.1 \ + --symfs "${stage_symfs}" -i "${data_file}" \ + > "${staged_session}/.perf-report.${part_name}.txt" \ + 2> "${staged_session}/.perf-report.${part_name}.stderr.txt" + ) & + batch_pids+=("$!") + ((index += 1)) + if (( ${#batch_pids[@]} >= parallel_jobs )); then + wait_for_batch + fi + done + wait_for_batch + (( parallel_failed )) && report_failed=1 + + : > "${staged_session}/perf-report.txt" + : > "${staged_session}/perf-report.stderr.txt" + if (( ${#staged_data_files[@]} > 1 )); then + echo "This logical capture used multiple perf recorder shards. Percentages below are local to each named raw shard; flamegraph.html and perf.script.gz aggregate samples from all shards." \ + >> "${staged_session}/perf-report.txt" + fi + index=0 + for data_file in "${staged_data_files[@]}"; do + printf -v part_name 'part%03d' "${index}" + printf '\n===== %s =====\n' "$(basename -- "${data_file}")" >> "${staged_session}/perf-report.txt" + cat "${staged_session}/.perf-report.${part_name}.txt" >> "${staged_session}/perf-report.txt" + if [[ -s "${staged_session}/.perf-report.${part_name}.stderr.txt" ]]; then + printf '\n===== %s =====\n' "$(basename -- "${data_file}")" >> "${staged_session}/perf-report.stderr.txt" + cat "${staged_session}/.perf-report.${part_name}.stderr.txt" >> "${staged_session}/perf-report.stderr.txt" + fi + ((index += 1)) + done + rm -f "${staged_session}"/.perf-report.part*.txt "${staged_session}"/.perf-report.part*.stderr.txt + if (( report_failed )); then + log "WARNING ${qid}: one or more text-report shards failed; continuing with flamegraph generation" + fi + + log "${qid}: parallel symbolized perf scripts" + parallel_failed=0 + batch_pids=() + part_files=() + index=0 + for data_file in "${staged_data_files[@]}"; do + printf -v part_name 'part%03d' "${index}" + part_file="${staged_session}/.perf-script.${part_name}" + part_files+=("${part_file}") + ( + perf script "${perf_inline_args[@]}" \ + --symfs "${stage_symfs}" -i "${data_file}" \ + > "${part_file}" \ + 2> "${part_file}.stderr.txt" + ) & + batch_pids+=("$!") + ((index += 1)) + if (( ${#batch_pids[@]} >= parallel_jobs )); then + wait_for_batch + fi + done + wait_for_batch + if (( parallel_failed )); then + : > "${staged_session}/perf-script.stderr.txt" + for part_file in "${part_files[@]}"; do + cat "${part_file}.stderr.txt" >> "${staged_session}/perf-script.stderr.txt" + done + log "ERROR ${qid}: perf script failed for one or more raw shards" + return 1 + fi + + : > "${script_file}" + : > "${staged_session}/perf-script.stderr.txt" + for part_file in "${part_files[@]}"; do + cat "${part_file}" >> "${script_file}" + cat "${part_file}.stderr.txt" >> "${staged_session}/perf-script.stderr.txt" + done + rm -f "${staged_session}"/.perf-script.part* "${staged_session}"/.perf-script.part*.stderr.txt + + : > "${generator_log}" + rm -f \ + "${staged_session}/flamegraph.html" \ + "${staged_session}/flamegraph-unavailable.txt" \ + "${staged_session}/stacks.folded" + + log "${qid}: bundled Python flamegraph renderer" + { + echo "Logical capture has ${#staged_data_files[@]} raw shard(s)." + echo "perf script inline-function expansion: ${inline_expansion}." + echo "Rendering the combined symbolized stream as a self-contained, click-to-zoom flamegraph." + } >> "${generator_log}" + if python3 "${FLAMEGRAPH_SCRIPT}" \ + --input "${script_file}" \ + --folded "${staged_session}/stacks.folded" \ + --output "${staged_session}/flamegraph.html" \ + --title "Presto worker ${worker_id}: ${qid}" \ + >> "${generator_log}" 2>&1; then + renderer_status="bundled-python" + else + { + echo "Flamegraph rendering failed; raw captures and the compressed symbolized stream were retained." + echo "See flamegraph-generator.txt and perf-script.stderr.txt." + } > "${staged_session}/flamegraph-unavailable.txt" + fi + + log "${qid}: compressing symbolized script" + gzip -1 -f "${script_file}" || { + log "ERROR ${qid}: could not compress perf.script" + return 1 + } + + { + echo "postprocess_version=2" + echo "postprocess_timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "postprocess_hostname=$(hostname)" + echo "postprocess_architecture=${current_arch}" + echo "capture_architecture=${captured_arch:-unknown}" + echo "postprocess_node_local_staging=1" + echo "postprocess_parallel_jobs=${parallel_jobs}" + echo "postprocess_perf_data_shards=${#staged_data_files[@]}" + echo "postprocess_inline_expansion=${inline_expansion}" + echo "postprocess_text_report_status=$([[ "${report_failed}" == "0" ]] && echo complete || echo partial)" + echo "flamegraph_renderer=${renderer_status}" + } > "${staged_session}/postprocess-metadata.txt" + + if [[ "${renderer_status}" != "bundled-python" ]]; then + publish_derived_files "${staged_session}" "${source_session}" 0 || return 1 + discard_staged_session "${staged_session}" + log "ERROR ${qid}: flamegraph rendering failed; diagnostics were published" + return 1 + fi + + publish_derived_files "${staged_session}" "${source_session}" 1 || return 1 + discard_staged_session "${staged_session}" + log "Completed ${qid}; flamegraph=${source_session}/flamegraph.html" +} + +log "Postprocessing on $(hostname) (${current_arch}); node-local staging=${stage_root}" +failed=0 +(( invalid_capture_count == 0 )) || failed=1 +for session_dir in "${session_dirs[@]}"; do + process_capture "${session_dir}" || failed=1 +done + +if (( failed )); then + log "One or more captures failed to post-process" + exit 1 +fi +log "All selected captures processed successfully" diff --git a/presto/slurm/presto-nvl72/run-presto-benchmarks.sh b/presto/slurm/presto-nvl72/run-presto-benchmarks.sh index bbb22135..8527585b 100755 --- a/presto/slurm/presto-nvl72/run-presto-benchmarks.sh +++ b/presto/slurm/presto-nvl72/run-presto-benchmarks.sh @@ -14,10 +14,51 @@ set -exuo pipefail source $SCRIPT_DIR/echo_helpers.sh source $SCRIPT_DIR/functions.sh -# Ensure metadata injection runs even if the script exits early (e.g. a worker -# fails to register). This guarantees benchmark_result.json always has a -# context block with image_digest before the results are copied out. -trap 'inject_benchmark_metadata' EXIT +function collect_coordinator_queries { + if [[ -x "${SCRIPT_DIR}/collect-coordinator-queries.sh" && -n "${COORD:-}" && -n "${PORT:-}" ]]; then + "${SCRIPT_DIR}/collect-coordinator-queries.sh" \ + --server "http://${COORD}:${PORT}" \ + --output-dir "${RESULT_DIR:-${SCRIPT_DIR}/result_dir}/coordinator_queries" || true + fi +} + +# Ensure metadata/log collection runs even if the script exits early (e.g. a +# query OOMs). DEBUG_HOLD_ON_FAILURE_SECONDS keeps the Slurm allocation alive +# long enough to inspect the coordinator and worker logs before teardown. +function on_exit { + local rc=$? + trap - EXIT + + # A perf capture may still be active when pytest or verification fails. + # Stop it and publish every node-local raw capture before Slurm tears down + # worker 0. Optional report generation also runs here, after all queries. + stop_perf_sampler || true + inject_benchmark_metadata || true + + if [[ "${rc}" != "0" ]]; then + collect_coordinator_queries + echo "Benchmark failed with exit code ${rc}; collecting logs before teardown." + collect_results || true + + if [[ -n "${DEBUG_HOLD_ON_FAILURE_SECONDS:-}" && "${DEBUG_HOLD_ON_FAILURE_SECONDS}" != "0" ]]; then + echo "Holding failed cluster for ${DEBUG_HOLD_ON_FAILURE_SECONDS}s." + echo "Coordinator: http://${COORD}:${PORT}" + echo "Logs: ${LOGS}" + echo "Result dir: ${RESULT_DIR:-${SCRIPT_DIR}/result_dir}" + sleep "${DEBUG_HOLD_ON_FAILURE_SECONDS}" || true + fi + fi + + if ! stop_cluster; then + echo "Cluster cleanup did not complete cleanly." >&2 + [[ "${rc}" == "0" ]] && rc=1 + fi + # Capture teardown and port-release diagnostics as well as startup logs. + collect_results || true + + exit "${rc}" +} +trap on_exit EXIT # ============================================================================== # Setup and Validation @@ -35,21 +76,31 @@ start_cluster # ============================================================================== echo "Running TPC-H queries (${NUM_ITERATIONS} iterations, scale factor ${SCALE_FACTOR})..." run_queries ${NUM_ITERATIONS} ${SCALE_FACTOR} +# Host perf must publish every node-local raw capture before worker teardown. +# Optional post-processing happens only now, after all measured queries. This is +# a no-op for ordinary and nsys runs. +stop_perf_sampler +collect_coordinator_queries +verify_cpu_ucx_runtime # ============================================================================== # Process Results # ============================================================================== echo "Processing results..." -mkdir -p ${SCRIPT_DIR}/result_dir -cp -r ${LOGS}/cli.log ${SCRIPT_DIR}/result_dir/summary.txt +mkdir -p "${RESULT_DIR:-${SCRIPT_DIR}/result_dir}" +cp -r "${LOGS}/cli.log" "${RESULT_DIR:-${SCRIPT_DIR}/result_dir}/summary.txt" echo "Collecting configs and logs into result directory..." collect_results wait_for_nsys_report_generation +stop_cluster +# Include explicit shutdown and port-release diagnostics in the immutable +# job-specific result directory. +collect_results echo "========================================" echo "Benchmark complete!" -echo "Results saved to: ${SCRIPT_DIR}/results_dir" +echo "Results saved to: ${RESULT_DIR:-${SCRIPT_DIR}/result_dir}" echo "Logs available at: ${LOGS}" echo "========================================"