From 24c924dbb2fc95bf7ba924404016c96316cfed55 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 12 Mar 2026 21:05:17 +0000 Subject: [PATCH 1/4] Initial plan From 544e6706a0f497fcf221e8e7821d8d8c1fcf1114 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 12 Mar 2026 21:10:15 +0000 Subject: [PATCH 2/4] Initial plan: run Example 7 GEMM all-scatter benchmark with IntelliKit profiling Co-authored-by: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com> --- .agents/skills/accordo/SKILL.md | 68 ++++++++++++++++++ .agents/skills/linex/SKILL.md | 98 ++++++++++++++++++++++++++ .agents/skills/metrix/SKILL.md | 76 ++++++++++++++++++++ .agents/skills/nexus/SKILL.md | 74 +++++++++++++++++++ .github/agents/skills/accordo/SKILL.md | 68 ++++++++++++++++++ .github/agents/skills/linex/SKILL.md | 98 ++++++++++++++++++++++++++ .github/agents/skills/metrix/SKILL.md | 76 ++++++++++++++++++++ .github/agents/skills/nexus/SKILL.md | 74 +++++++++++++++++++ .intellikit | 1 + 9 files changed, 633 insertions(+) create mode 100644 .agents/skills/accordo/SKILL.md create mode 100644 .agents/skills/linex/SKILL.md create mode 100644 .agents/skills/metrix/SKILL.md create mode 100644 .agents/skills/nexus/SKILL.md create mode 100644 .github/agents/skills/accordo/SKILL.md create mode 100644 .github/agents/skills/linex/SKILL.md create mode 100644 .github/agents/skills/metrix/SKILL.md create mode 100644 .github/agents/skills/nexus/SKILL.md create mode 160000 .intellikit diff --git a/.agents/skills/accordo/SKILL.md b/.agents/skills/accordo/SKILL.md new file mode 100644 index 000000000..808c73ef9 --- /dev/null +++ b/.agents/skills/accordo/SKILL.md @@ -0,0 +1,68 @@ +--- +name: accordo-validation +description: Validate GPU kernel correctness by comparing reference and optimized outputs. Use when verifying that an optimized or modified kernel matches a reference implementation. +--- + +# Accordo: GPU Kernel Validation + +Capture and compare kernel outputs from reference and optimized binaries to validate correctness. Uses kernelDB for automatic kernel extraction; supports configurable tolerance and execution-time comparison. + +## When to Use + +- User has a reference and an optimized (or modified) GPU kernel and wants to check they produce the same results +- Regression testing after kernel or build changes +- Validating multiple optimization variants against one baseline + +## Instructions + +1. **Require two or more binaries:** one reference (e.g. `./app_ref`) and one or more to validate (e.g. `./app_opt`). All must expose the same kernel by name. +2. **Ensure binaries are built with debug symbols** (`-g`) so kernel arguments can be extracted. +3. **Choose execution path:** + - If an Accordo MCP server is available, call its `validate_kernel_correctness` tool, which performs capture-and-compare with the same semantics described below. + - Otherwise use the Python API from the environment where Accordo is installed. + +### Python API + +```python +from accordo import Accordo + +# Validator for the kernel to validate (binary used to extract signature) +validator = Accordo(binary="./app_ref", kernel_name="reduce_sum") + +# Optional: set working directory if binaries expect it +validator = Accordo(binary="./app_ref", kernel_name="reduce_sum", working_directory="./run") + +# Capture snapshots +ref = validator.capture_snapshot(binary="./app_ref") +opt = validator.capture_snapshot(binary="./app_opt") + +# Compare with tolerance (default 1e-6) +result = validator.compare_snapshots(ref, opt, tolerance=1e-6) + +if result.is_valid: + print("PASS:", result.num_arrays_validated, "arrays matched") +else: + print(result.summary()) +``` + +For multiple optimizations, capture the reference once and compare each optimized snapshot against it. + +### Snapshot and result attributes + +- **Snapshot:** `arrays`, `execution_time_ms`, `grid_size`, `block_size` +- **ValidationResult:** `is_valid`, `num_arrays_validated`, `num_mismatches`, `mismatches`, `success_rate`; use `summary()` for a human-readable report. + +## Workflow + +1. Build reference and optimized binaries with the same kernel name and `-g`. +2. Create an `Accordo(binary=ref_binary, kernel_name="...")` validator; set `working_directory` if needed. +3. Capture reference snapshot with `capture_snapshot(binary=ref_binary)`. +4. For each variant, capture with `capture_snapshot(binary=opt_binary)` and compare with `compare_snapshots(ref, opt, tolerance=...)`. +5. If `result.is_valid` is false, use `result.summary()` and `result.mismatches` to diagnose. +6. Use relative paths for binaries and working directory so the skill is portable. + +## Notes + +- kernelDB is used automatically; no separate kernelDB setup is required when using the Python API. +- Increase `tolerance` for floating-point comparisons when appropriate (e.g. 1e-4 or 1e-5 for single precision). +- Use `timeout_seconds` in `capture_snapshot` if the run may hang. diff --git a/.agents/skills/linex/SKILL.md b/.agents/skills/linex/SKILL.md new file mode 100644 index 000000000..dca5b7d6d --- /dev/null +++ b/.agents/skills/linex/SKILL.md @@ -0,0 +1,98 @@ +--- +name: linex-profiling +description: Profile GPU kernels at source-line granularity with cycle-level timing and stall analysis. Use when identifying performance hotspots at the source code level or analyzing instruction-level metrics mapped to source lines. +--- + +# Linex: Source-Level GPU Performance Profiling + +Map GPU performance metrics to your source code lines. Get cycle-level timing, stall analysis, and instruction-level metrics for each line of source code. + +## When to Use + +- User asks to profile a GPU application at source-line granularity +- Need to identify which specific lines of code are performance bottlenecks +- Analyzing stall patterns and execution bottlenecks at the source level +- Understanding cycle-level timing for each line of code +- Instruction-level analysis mapped to source lines + +## Instructions + +1. **Ensure the target runs on AMD ROCm 7.0+** with `rocprofv3` available. +2. **Kernels must be compiled with `-g`** (debug symbols) for source mapping. +3. **Choose execution path:** + - If a Linex MCP server is available, use its MCP tools: + - `profile_application` to run and profile a target application with the options below. + - `analyze_instruction_hotspots` to perform instruction-level hotspot analysis on collected profiles. + - Otherwise use the Python API from the environment where Linex is installed. + +### Python API + +```python +from linex import Linex + +profiler = Linex( + target_cu=0, # Target compute unit + shader_engine_mask="0xFFFFFFFF", # All shader engines + activity=10, # Activity counter polling +) + +profiler.profile("./my_app", kernel_filter="my_kernel") + +# Show hotspots (sorted by total_cycles) +for line in profiler.source_lines[:5]: + print(f"{line.file}:{line.line_number}") + print(f" {line.total_cycles:,} cycles ({line.stall_percent:.1f}% stalled)") + print(f" Executed {line.execution_count} times") + +# Find memory-bound lines +memory_bound = [ + l for l in profiler.source_lines + if l.stall_percent > 50 +] + +# Instruction-level analysis +for line in profiler.source_lines[:1]: + for inst in line.instructions: + print(f"{inst.isa}: {inst.latency_cycles} cycles") +``` + +### SourceLine Properties + +- `file` - Source file path +- `line_number` - Line number +- `total_cycles` - Sum of all instruction cycles +- `stall_cycles` - Cycles spent waiting +- `idle_cycles` - Cycles slot was idle +- `execution_count` - Total executions +- `instructions` - List of ISA instructions +- `stall_percent` - Convenience: stall_cycles / total_cycles * 100 + +### InstructionData Properties + +- `isa` - ISA instruction text +- `latency_cycles` - Total cycles for this instruction +- `stall_cycles` - Cycles spent waiting +- `idle_cycles` - Cycles slot was idle +- `execution_count` - How many times it ran +- `instruction_address` - Virtual address in GPU memory +- `file` - Parsed from source_location +- `line` - Parsed from source_location +- `stall_percent` - Convenience: stall_cycles / latency_cycles * 100 + +## Workflow + +1. Ensure the target binary is built with `-g` (debug symbols) for source mapping. +2. Create a `Linex()` profiler; optionally set `target_cu`, `shader_engine_mask`, or `activity`. +3. Call `profiler.profile(command, kernel_filter=...)` to run profiling. +4. Access `profiler.source_lines` (sorted by total_cycles) to find hotspots. +5. Use `line.stall_percent` to identify memory-bound or dependency-bound lines. +6. Drill down into `line.instructions` for instruction-level analysis. +7. Use relative paths for the target binary so the skill is portable. + +## Notes + +- Requires ROCm 7.0+ with `rocprofv3` support. +- Source mapping requires kernels compiled with `-g` (debug symbols). +- `source_lines` are automatically sorted by `total_cycles` (descending). +- Use `kernel_filter` to profile specific kernels by name (regex pattern). +- For Triton or other frameworks, ensure debug symbols are available in the compiled output. diff --git a/.agents/skills/metrix/SKILL.md b/.agents/skills/metrix/SKILL.md new file mode 100644 index 000000000..9a5564ece --- /dev/null +++ b/.agents/skills/metrix/SKILL.md @@ -0,0 +1,76 @@ +--- +name: metrix-profiling +description: Profile GPU kernels when performance analysis or optimization is required. Use for AMD ROCm GPU metrics, bandwidth, cache hit rates, coalescing, or kernel timing. +--- + +# Metrix: GPU Profiling + +Profile AMD GPU kernels and get human-readable metrics (bandwidth, cache, coalescing, FLOPS). Architecture is auto-detected. + +## When to Use + +- User asks to profile a GPU application or kernel +- Performance analysis, optimization, or bottleneck investigation +- Need HBM/L2/L1 bandwidth, hit rates, or compute metrics +- Need timing-only runs (fast, no hardware counters) + +## Instructions + +1. **Ensure the target runs on AMD ROCm** (e.g. `hipcc`-built binary or Python script that launches HIP/ROCm kernels). +2. **Choose execution path:** + - If a Metrix MCP server is available, use its profile tool with the same options below. + - Otherwise run the CLI or Python API from the environment where Metrix is installed. + +### CLI + +From the project or install prefix: + +```bash +# Profile with all metrics (auto-detected arch) +metrix ./my_app + +# Time only (fast, no counters) +metrix --time-only -n 10 ./my_app + +# Filter kernels by name +metrix --kernel matmul ./my_app + +# Specific metrics +metrix --metrics memory.l2_hit_rate,memory.coalescing_efficiency,compute.total_flops ./my_app + +# Save to JSON/CSV +metrix -o results.json ./my_app +``` + +Options: `--profile` (quick|memory|compute), `--metrics` (comma-separated), `--time-only`, `--kernel` (substring), `--num-replays`/`-n`, `--output`/`-o`, `--top K`, `--aggregate`, `--log`/`-l` (debug|info|warning|error), `--quiet`. + +### Python API + +```python +from metrix import Metrix + +profiler = Metrix() +results = profiler.profile("./my_app", num_replays=5) + +for kernel in results.kernels: + print(kernel.name, kernel.duration_us.avg) + for metric, stats in kernel.metrics.items(): + print(f" {metric}: {stats.avg}") +``` + +Use `metrics=[...]` for a subset; omit for all metrics. Use `cwd` when the binary expects a specific working directory. + +## Workflow + +1. Identify the executable or script to profile (e.g. `./app` or `python run_kernels.py`). +2. If only timing is needed, use `--time-only` for speed. +3. If full metrics are needed, run `metrix ./app` (or MCP equivalent); optionally restrict with `--kernel` or `--metrics`. +4. Interpret results: low L2 hit rate, low coalescing, or low HBM utilization suggest optimization targets. +5. For automation or tooling, use `-o results.json` and parse the JSON output. + +## Key Metrics (reference) + +- **Memory:** `memory.hbm_bandwidth_utilization`, `memory.l2_hit_rate`, `memory.l1_hit_rate`, `memory.coalescing_efficiency`, `memory.global_load_efficiency`, `memory.lds_bank_conflicts`, `memory.atomic_latency` +- **Compute:** `compute.total_flops`, `compute.hbm_gflops`, `compute.hbm_arithmetic_intensity`, `compute.l2_arithmetic_intensity`, `compute.l1_arithmetic_intensity` + +Use relative paths for the target binary and output files so the skill is portable across environments. diff --git a/.agents/skills/nexus/SKILL.md b/.agents/skills/nexus/SKILL.md new file mode 100644 index 000000000..ad714bc4d --- /dev/null +++ b/.agents/skills/nexus/SKILL.md @@ -0,0 +1,74 @@ +--- +name: nexus-trace +description: Extract GPU kernel assembly and HIP source from HSA packet traces. Use when analyzing what code ran on the GPU, debugging kernel dispatch, or inspecting assembly and source mapping. +--- + +# Nexus: HSA Packet Source Code Extractor + +Intercepts HSA packets from a running process and extracts, per kernel, assembly and HIP source into a structured trace (e.g. JSON). Use for kernel-level inspection and assembly/source correlation. + +## When to Use + +- User needs to see which kernels ran and their assembly or HIP source +- Debugging or analyzing GPU dispatch and code generation +- Inspecting assembly-to-source mapping for a HIP (or ROCm) application + +## Instructions + +1. **Ensure the target runs on AMD ROCm** and uses HSA (e.g. HIP application or ROCm runtime). +2. **Choose execution path:** + - If a Nexus MCP server is available, use its tools: `list_kernels` to enumerate kernels in a trace, and `extract_kernel_code` to get assembly and HIP/source mapping (signature, files, lines). See `nexus/nexus/mcp/server.py` for tool parameters and schemas. + - Otherwise use the Python API from the environment where Nexus is installed. + +### Python API (recommended when no MCP) + +```python +from nexus import Nexus + +nexus = Nexus(log_level=1) +trace = nexus.run(["python", "my_gpu_script.py"]) + +# Or run a binary: +# trace = nexus.run(["./my_hip_app"]) + +for kernel in trace: + print(kernel.name, len(kernel.assembly), "instructions") + for i, asm_line in enumerate(kernel.assembly, 1): + print(f" {i}. {asm_line}") + for line_no, hip_line in zip(kernel.lines or range(1, len(kernel.hip)+1), kernel.hip): + print(f" {line_no}: {hip_line}") + +# Access by kernel name +k = trace["vector_add(float const*, float const*, float*, int)"] +print(k.assembly, k.hip, k.signature, k.files, k.lines) + +# Save/load trace +trace.save("trace.json") +loaded = Nexus.load("trace.json") +``` + +Set `log_level` (0–4) to control verbosity. Use relative paths for the run command and output file so the skill is portable. + +### Environment-based usage (no Python API) + +When the process cannot be launched via `nexus.run()`: + +1. Set `HSA_TOOLS_LIB` to the Nexus shared library path (e.g. `build/lib/libnexus.so` or the installed path). +2. Set `NEXUS_OUTPUT_FILE` to the output JSON path. +3. Set `NEXUS_LOG_LEVEL` (0–4) if needed. +4. Run the application as usual; it will be traced and the output file will contain the kernel data. + +Optional: `NEXUS_EXTRA_SEARCH_PREFIX` (colon-separated) for HIP source search; `TRITON_DISABLE_LINE_INFO=0` for Triton kernel line info. + +## Workflow + +1. Identify the command that runs the GPU workload (e.g. `python script.py` or `./app`). +2. If using the Python API: create `Nexus(log_level=...)`, call `nexus.run([...])`, then iterate `trace` and optionally `trace.save(...)`. +3. If using the env method: set `HSA_TOOLS_LIB` and `NEXUS_OUTPUT_FILE`, then run the app; open the JSON and parse the `kernels` structure. +4. Use kernel `signature`, `assembly`, `hip`, `files`, and `lines` to analyze what ran and map assembly back to source. +5. Use relative paths for commands and output files. + +## Notes + +- Nexus is intended for research/analysis; ensure the target environment has the Nexus library and compatible ROCm/HSA stack. +- For Triton kernels, enable line info via `TRITON_DISABLE_LINE_INFO=0` when using the Python API. diff --git a/.github/agents/skills/accordo/SKILL.md b/.github/agents/skills/accordo/SKILL.md new file mode 100644 index 000000000..808c73ef9 --- /dev/null +++ b/.github/agents/skills/accordo/SKILL.md @@ -0,0 +1,68 @@ +--- +name: accordo-validation +description: Validate GPU kernel correctness by comparing reference and optimized outputs. Use when verifying that an optimized or modified kernel matches a reference implementation. +--- + +# Accordo: GPU Kernel Validation + +Capture and compare kernel outputs from reference and optimized binaries to validate correctness. Uses kernelDB for automatic kernel extraction; supports configurable tolerance and execution-time comparison. + +## When to Use + +- User has a reference and an optimized (or modified) GPU kernel and wants to check they produce the same results +- Regression testing after kernel or build changes +- Validating multiple optimization variants against one baseline + +## Instructions + +1. **Require two or more binaries:** one reference (e.g. `./app_ref`) and one or more to validate (e.g. `./app_opt`). All must expose the same kernel by name. +2. **Ensure binaries are built with debug symbols** (`-g`) so kernel arguments can be extracted. +3. **Choose execution path:** + - If an Accordo MCP server is available, call its `validate_kernel_correctness` tool, which performs capture-and-compare with the same semantics described below. + - Otherwise use the Python API from the environment where Accordo is installed. + +### Python API + +```python +from accordo import Accordo + +# Validator for the kernel to validate (binary used to extract signature) +validator = Accordo(binary="./app_ref", kernel_name="reduce_sum") + +# Optional: set working directory if binaries expect it +validator = Accordo(binary="./app_ref", kernel_name="reduce_sum", working_directory="./run") + +# Capture snapshots +ref = validator.capture_snapshot(binary="./app_ref") +opt = validator.capture_snapshot(binary="./app_opt") + +# Compare with tolerance (default 1e-6) +result = validator.compare_snapshots(ref, opt, tolerance=1e-6) + +if result.is_valid: + print("PASS:", result.num_arrays_validated, "arrays matched") +else: + print(result.summary()) +``` + +For multiple optimizations, capture the reference once and compare each optimized snapshot against it. + +### Snapshot and result attributes + +- **Snapshot:** `arrays`, `execution_time_ms`, `grid_size`, `block_size` +- **ValidationResult:** `is_valid`, `num_arrays_validated`, `num_mismatches`, `mismatches`, `success_rate`; use `summary()` for a human-readable report. + +## Workflow + +1. Build reference and optimized binaries with the same kernel name and `-g`. +2. Create an `Accordo(binary=ref_binary, kernel_name="...")` validator; set `working_directory` if needed. +3. Capture reference snapshot with `capture_snapshot(binary=ref_binary)`. +4. For each variant, capture with `capture_snapshot(binary=opt_binary)` and compare with `compare_snapshots(ref, opt, tolerance=...)`. +5. If `result.is_valid` is false, use `result.summary()` and `result.mismatches` to diagnose. +6. Use relative paths for binaries and working directory so the skill is portable. + +## Notes + +- kernelDB is used automatically; no separate kernelDB setup is required when using the Python API. +- Increase `tolerance` for floating-point comparisons when appropriate (e.g. 1e-4 or 1e-5 for single precision). +- Use `timeout_seconds` in `capture_snapshot` if the run may hang. diff --git a/.github/agents/skills/linex/SKILL.md b/.github/agents/skills/linex/SKILL.md new file mode 100644 index 000000000..dca5b7d6d --- /dev/null +++ b/.github/agents/skills/linex/SKILL.md @@ -0,0 +1,98 @@ +--- +name: linex-profiling +description: Profile GPU kernels at source-line granularity with cycle-level timing and stall analysis. Use when identifying performance hotspots at the source code level or analyzing instruction-level metrics mapped to source lines. +--- + +# Linex: Source-Level GPU Performance Profiling + +Map GPU performance metrics to your source code lines. Get cycle-level timing, stall analysis, and instruction-level metrics for each line of source code. + +## When to Use + +- User asks to profile a GPU application at source-line granularity +- Need to identify which specific lines of code are performance bottlenecks +- Analyzing stall patterns and execution bottlenecks at the source level +- Understanding cycle-level timing for each line of code +- Instruction-level analysis mapped to source lines + +## Instructions + +1. **Ensure the target runs on AMD ROCm 7.0+** with `rocprofv3` available. +2. **Kernels must be compiled with `-g`** (debug symbols) for source mapping. +3. **Choose execution path:** + - If a Linex MCP server is available, use its MCP tools: + - `profile_application` to run and profile a target application with the options below. + - `analyze_instruction_hotspots` to perform instruction-level hotspot analysis on collected profiles. + - Otherwise use the Python API from the environment where Linex is installed. + +### Python API + +```python +from linex import Linex + +profiler = Linex( + target_cu=0, # Target compute unit + shader_engine_mask="0xFFFFFFFF", # All shader engines + activity=10, # Activity counter polling +) + +profiler.profile("./my_app", kernel_filter="my_kernel") + +# Show hotspots (sorted by total_cycles) +for line in profiler.source_lines[:5]: + print(f"{line.file}:{line.line_number}") + print(f" {line.total_cycles:,} cycles ({line.stall_percent:.1f}% stalled)") + print(f" Executed {line.execution_count} times") + +# Find memory-bound lines +memory_bound = [ + l for l in profiler.source_lines + if l.stall_percent > 50 +] + +# Instruction-level analysis +for line in profiler.source_lines[:1]: + for inst in line.instructions: + print(f"{inst.isa}: {inst.latency_cycles} cycles") +``` + +### SourceLine Properties + +- `file` - Source file path +- `line_number` - Line number +- `total_cycles` - Sum of all instruction cycles +- `stall_cycles` - Cycles spent waiting +- `idle_cycles` - Cycles slot was idle +- `execution_count` - Total executions +- `instructions` - List of ISA instructions +- `stall_percent` - Convenience: stall_cycles / total_cycles * 100 + +### InstructionData Properties + +- `isa` - ISA instruction text +- `latency_cycles` - Total cycles for this instruction +- `stall_cycles` - Cycles spent waiting +- `idle_cycles` - Cycles slot was idle +- `execution_count` - How many times it ran +- `instruction_address` - Virtual address in GPU memory +- `file` - Parsed from source_location +- `line` - Parsed from source_location +- `stall_percent` - Convenience: stall_cycles / latency_cycles * 100 + +## Workflow + +1. Ensure the target binary is built with `-g` (debug symbols) for source mapping. +2. Create a `Linex()` profiler; optionally set `target_cu`, `shader_engine_mask`, or `activity`. +3. Call `profiler.profile(command, kernel_filter=...)` to run profiling. +4. Access `profiler.source_lines` (sorted by total_cycles) to find hotspots. +5. Use `line.stall_percent` to identify memory-bound or dependency-bound lines. +6. Drill down into `line.instructions` for instruction-level analysis. +7. Use relative paths for the target binary so the skill is portable. + +## Notes + +- Requires ROCm 7.0+ with `rocprofv3` support. +- Source mapping requires kernels compiled with `-g` (debug symbols). +- `source_lines` are automatically sorted by `total_cycles` (descending). +- Use `kernel_filter` to profile specific kernels by name (regex pattern). +- For Triton or other frameworks, ensure debug symbols are available in the compiled output. diff --git a/.github/agents/skills/metrix/SKILL.md b/.github/agents/skills/metrix/SKILL.md new file mode 100644 index 000000000..9a5564ece --- /dev/null +++ b/.github/agents/skills/metrix/SKILL.md @@ -0,0 +1,76 @@ +--- +name: metrix-profiling +description: Profile GPU kernels when performance analysis or optimization is required. Use for AMD ROCm GPU metrics, bandwidth, cache hit rates, coalescing, or kernel timing. +--- + +# Metrix: GPU Profiling + +Profile AMD GPU kernels and get human-readable metrics (bandwidth, cache, coalescing, FLOPS). Architecture is auto-detected. + +## When to Use + +- User asks to profile a GPU application or kernel +- Performance analysis, optimization, or bottleneck investigation +- Need HBM/L2/L1 bandwidth, hit rates, or compute metrics +- Need timing-only runs (fast, no hardware counters) + +## Instructions + +1. **Ensure the target runs on AMD ROCm** (e.g. `hipcc`-built binary or Python script that launches HIP/ROCm kernels). +2. **Choose execution path:** + - If a Metrix MCP server is available, use its profile tool with the same options below. + - Otherwise run the CLI or Python API from the environment where Metrix is installed. + +### CLI + +From the project or install prefix: + +```bash +# Profile with all metrics (auto-detected arch) +metrix ./my_app + +# Time only (fast, no counters) +metrix --time-only -n 10 ./my_app + +# Filter kernels by name +metrix --kernel matmul ./my_app + +# Specific metrics +metrix --metrics memory.l2_hit_rate,memory.coalescing_efficiency,compute.total_flops ./my_app + +# Save to JSON/CSV +metrix -o results.json ./my_app +``` + +Options: `--profile` (quick|memory|compute), `--metrics` (comma-separated), `--time-only`, `--kernel` (substring), `--num-replays`/`-n`, `--output`/`-o`, `--top K`, `--aggregate`, `--log`/`-l` (debug|info|warning|error), `--quiet`. + +### Python API + +```python +from metrix import Metrix + +profiler = Metrix() +results = profiler.profile("./my_app", num_replays=5) + +for kernel in results.kernels: + print(kernel.name, kernel.duration_us.avg) + for metric, stats in kernel.metrics.items(): + print(f" {metric}: {stats.avg}") +``` + +Use `metrics=[...]` for a subset; omit for all metrics. Use `cwd` when the binary expects a specific working directory. + +## Workflow + +1. Identify the executable or script to profile (e.g. `./app` or `python run_kernels.py`). +2. If only timing is needed, use `--time-only` for speed. +3. If full metrics are needed, run `metrix ./app` (or MCP equivalent); optionally restrict with `--kernel` or `--metrics`. +4. Interpret results: low L2 hit rate, low coalescing, or low HBM utilization suggest optimization targets. +5. For automation or tooling, use `-o results.json` and parse the JSON output. + +## Key Metrics (reference) + +- **Memory:** `memory.hbm_bandwidth_utilization`, `memory.l2_hit_rate`, `memory.l1_hit_rate`, `memory.coalescing_efficiency`, `memory.global_load_efficiency`, `memory.lds_bank_conflicts`, `memory.atomic_latency` +- **Compute:** `compute.total_flops`, `compute.hbm_gflops`, `compute.hbm_arithmetic_intensity`, `compute.l2_arithmetic_intensity`, `compute.l1_arithmetic_intensity` + +Use relative paths for the target binary and output files so the skill is portable across environments. diff --git a/.github/agents/skills/nexus/SKILL.md b/.github/agents/skills/nexus/SKILL.md new file mode 100644 index 000000000..ad714bc4d --- /dev/null +++ b/.github/agents/skills/nexus/SKILL.md @@ -0,0 +1,74 @@ +--- +name: nexus-trace +description: Extract GPU kernel assembly and HIP source from HSA packet traces. Use when analyzing what code ran on the GPU, debugging kernel dispatch, or inspecting assembly and source mapping. +--- + +# Nexus: HSA Packet Source Code Extractor + +Intercepts HSA packets from a running process and extracts, per kernel, assembly and HIP source into a structured trace (e.g. JSON). Use for kernel-level inspection and assembly/source correlation. + +## When to Use + +- User needs to see which kernels ran and their assembly or HIP source +- Debugging or analyzing GPU dispatch and code generation +- Inspecting assembly-to-source mapping for a HIP (or ROCm) application + +## Instructions + +1. **Ensure the target runs on AMD ROCm** and uses HSA (e.g. HIP application or ROCm runtime). +2. **Choose execution path:** + - If a Nexus MCP server is available, use its tools: `list_kernels` to enumerate kernels in a trace, and `extract_kernel_code` to get assembly and HIP/source mapping (signature, files, lines). See `nexus/nexus/mcp/server.py` for tool parameters and schemas. + - Otherwise use the Python API from the environment where Nexus is installed. + +### Python API (recommended when no MCP) + +```python +from nexus import Nexus + +nexus = Nexus(log_level=1) +trace = nexus.run(["python", "my_gpu_script.py"]) + +# Or run a binary: +# trace = nexus.run(["./my_hip_app"]) + +for kernel in trace: + print(kernel.name, len(kernel.assembly), "instructions") + for i, asm_line in enumerate(kernel.assembly, 1): + print(f" {i}. {asm_line}") + for line_no, hip_line in zip(kernel.lines or range(1, len(kernel.hip)+1), kernel.hip): + print(f" {line_no}: {hip_line}") + +# Access by kernel name +k = trace["vector_add(float const*, float const*, float*, int)"] +print(k.assembly, k.hip, k.signature, k.files, k.lines) + +# Save/load trace +trace.save("trace.json") +loaded = Nexus.load("trace.json") +``` + +Set `log_level` (0–4) to control verbosity. Use relative paths for the run command and output file so the skill is portable. + +### Environment-based usage (no Python API) + +When the process cannot be launched via `nexus.run()`: + +1. Set `HSA_TOOLS_LIB` to the Nexus shared library path (e.g. `build/lib/libnexus.so` or the installed path). +2. Set `NEXUS_OUTPUT_FILE` to the output JSON path. +3. Set `NEXUS_LOG_LEVEL` (0–4) if needed. +4. Run the application as usual; it will be traced and the output file will contain the kernel data. + +Optional: `NEXUS_EXTRA_SEARCH_PREFIX` (colon-separated) for HIP source search; `TRITON_DISABLE_LINE_INFO=0` for Triton kernel line info. + +## Workflow + +1. Identify the command that runs the GPU workload (e.g. `python script.py` or `./app`). +2. If using the Python API: create `Nexus(log_level=...)`, call `nexus.run([...])`, then iterate `trace` and optionally `trace.save(...)`. +3. If using the env method: set `HSA_TOOLS_LIB` and `NEXUS_OUTPUT_FILE`, then run the app; open the JSON and parse the `kernels` structure. +4. Use kernel `signature`, `assembly`, `hip`, `files`, and `lines` to analyze what ran and map assembly back to source. +5. Use relative paths for commands and output files. + +## Notes + +- Nexus is intended for research/analysis; ensure the target environment has the Nexus library and compatible ROCm/HSA stack. +- For Triton kernels, enable line info via `TRITON_DISABLE_LINE_INFO=0` when using the Python API. diff --git a/.intellikit b/.intellikit new file mode 160000 index 000000000..f69ecf13c --- /dev/null +++ b/.intellikit @@ -0,0 +1 @@ +Subproject commit f69ecf13c59fe73e58fb37ee2f6ea02fafd22444 From 674ebd209e920c949ae98547c61a9e38ed7c2a85 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 12 Mar 2026 21:19:49 +0000 Subject: [PATCH 3/4] Add IntelliKit Metrix profiling for Example 07 GEMM All-Scatter benchmark Co-authored-by: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com> --- .agents/skills/accordo/SKILL.md | 68 ------- .agents/skills/linex/SKILL.md | 98 ---------- .agents/skills/metrix/SKILL.md | 76 ------- .agents/skills/nexus/SKILL.md | 74 ------- .gitignore | 4 +- .../profiling/profile_with_metrix.py | 185 ++++++++++++++++++ 6 files changed, 188 insertions(+), 317 deletions(-) delete mode 100644 .agents/skills/accordo/SKILL.md delete mode 100644 .agents/skills/linex/SKILL.md delete mode 100644 .agents/skills/metrix/SKILL.md delete mode 100644 .agents/skills/nexus/SKILL.md create mode 100644 examples/07_gemm_all_scatter/profiling/profile_with_metrix.py diff --git a/.agents/skills/accordo/SKILL.md b/.agents/skills/accordo/SKILL.md deleted file mode 100644 index 808c73ef9..000000000 --- a/.agents/skills/accordo/SKILL.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -name: accordo-validation -description: Validate GPU kernel correctness by comparing reference and optimized outputs. Use when verifying that an optimized or modified kernel matches a reference implementation. ---- - -# Accordo: GPU Kernel Validation - -Capture and compare kernel outputs from reference and optimized binaries to validate correctness. Uses kernelDB for automatic kernel extraction; supports configurable tolerance and execution-time comparison. - -## When to Use - -- User has a reference and an optimized (or modified) GPU kernel and wants to check they produce the same results -- Regression testing after kernel or build changes -- Validating multiple optimization variants against one baseline - -## Instructions - -1. **Require two or more binaries:** one reference (e.g. `./app_ref`) and one or more to validate (e.g. `./app_opt`). All must expose the same kernel by name. -2. **Ensure binaries are built with debug symbols** (`-g`) so kernel arguments can be extracted. -3. **Choose execution path:** - - If an Accordo MCP server is available, call its `validate_kernel_correctness` tool, which performs capture-and-compare with the same semantics described below. - - Otherwise use the Python API from the environment where Accordo is installed. - -### Python API - -```python -from accordo import Accordo - -# Validator for the kernel to validate (binary used to extract signature) -validator = Accordo(binary="./app_ref", kernel_name="reduce_sum") - -# Optional: set working directory if binaries expect it -validator = Accordo(binary="./app_ref", kernel_name="reduce_sum", working_directory="./run") - -# Capture snapshots -ref = validator.capture_snapshot(binary="./app_ref") -opt = validator.capture_snapshot(binary="./app_opt") - -# Compare with tolerance (default 1e-6) -result = validator.compare_snapshots(ref, opt, tolerance=1e-6) - -if result.is_valid: - print("PASS:", result.num_arrays_validated, "arrays matched") -else: - print(result.summary()) -``` - -For multiple optimizations, capture the reference once and compare each optimized snapshot against it. - -### Snapshot and result attributes - -- **Snapshot:** `arrays`, `execution_time_ms`, `grid_size`, `block_size` -- **ValidationResult:** `is_valid`, `num_arrays_validated`, `num_mismatches`, `mismatches`, `success_rate`; use `summary()` for a human-readable report. - -## Workflow - -1. Build reference and optimized binaries with the same kernel name and `-g`. -2. Create an `Accordo(binary=ref_binary, kernel_name="...")` validator; set `working_directory` if needed. -3. Capture reference snapshot with `capture_snapshot(binary=ref_binary)`. -4. For each variant, capture with `capture_snapshot(binary=opt_binary)` and compare with `compare_snapshots(ref, opt, tolerance=...)`. -5. If `result.is_valid` is false, use `result.summary()` and `result.mismatches` to diagnose. -6. Use relative paths for binaries and working directory so the skill is portable. - -## Notes - -- kernelDB is used automatically; no separate kernelDB setup is required when using the Python API. -- Increase `tolerance` for floating-point comparisons when appropriate (e.g. 1e-4 or 1e-5 for single precision). -- Use `timeout_seconds` in `capture_snapshot` if the run may hang. diff --git a/.agents/skills/linex/SKILL.md b/.agents/skills/linex/SKILL.md deleted file mode 100644 index dca5b7d6d..000000000 --- a/.agents/skills/linex/SKILL.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -name: linex-profiling -description: Profile GPU kernels at source-line granularity with cycle-level timing and stall analysis. Use when identifying performance hotspots at the source code level or analyzing instruction-level metrics mapped to source lines. ---- - -# Linex: Source-Level GPU Performance Profiling - -Map GPU performance metrics to your source code lines. Get cycle-level timing, stall analysis, and instruction-level metrics for each line of source code. - -## When to Use - -- User asks to profile a GPU application at source-line granularity -- Need to identify which specific lines of code are performance bottlenecks -- Analyzing stall patterns and execution bottlenecks at the source level -- Understanding cycle-level timing for each line of code -- Instruction-level analysis mapped to source lines - -## Instructions - -1. **Ensure the target runs on AMD ROCm 7.0+** with `rocprofv3` available. -2. **Kernels must be compiled with `-g`** (debug symbols) for source mapping. -3. **Choose execution path:** - - If a Linex MCP server is available, use its MCP tools: - - `profile_application` to run and profile a target application with the options below. - - `analyze_instruction_hotspots` to perform instruction-level hotspot analysis on collected profiles. - - Otherwise use the Python API from the environment where Linex is installed. - -### Python API - -```python -from linex import Linex - -profiler = Linex( - target_cu=0, # Target compute unit - shader_engine_mask="0xFFFFFFFF", # All shader engines - activity=10, # Activity counter polling -) - -profiler.profile("./my_app", kernel_filter="my_kernel") - -# Show hotspots (sorted by total_cycles) -for line in profiler.source_lines[:5]: - print(f"{line.file}:{line.line_number}") - print(f" {line.total_cycles:,} cycles ({line.stall_percent:.1f}% stalled)") - print(f" Executed {line.execution_count} times") - -# Find memory-bound lines -memory_bound = [ - l for l in profiler.source_lines - if l.stall_percent > 50 -] - -# Instruction-level analysis -for line in profiler.source_lines[:1]: - for inst in line.instructions: - print(f"{inst.isa}: {inst.latency_cycles} cycles") -``` - -### SourceLine Properties - -- `file` - Source file path -- `line_number` - Line number -- `total_cycles` - Sum of all instruction cycles -- `stall_cycles` - Cycles spent waiting -- `idle_cycles` - Cycles slot was idle -- `execution_count` - Total executions -- `instructions` - List of ISA instructions -- `stall_percent` - Convenience: stall_cycles / total_cycles * 100 - -### InstructionData Properties - -- `isa` - ISA instruction text -- `latency_cycles` - Total cycles for this instruction -- `stall_cycles` - Cycles spent waiting -- `idle_cycles` - Cycles slot was idle -- `execution_count` - How many times it ran -- `instruction_address` - Virtual address in GPU memory -- `file` - Parsed from source_location -- `line` - Parsed from source_location -- `stall_percent` - Convenience: stall_cycles / latency_cycles * 100 - -## Workflow - -1. Ensure the target binary is built with `-g` (debug symbols) for source mapping. -2. Create a `Linex()` profiler; optionally set `target_cu`, `shader_engine_mask`, or `activity`. -3. Call `profiler.profile(command, kernel_filter=...)` to run profiling. -4. Access `profiler.source_lines` (sorted by total_cycles) to find hotspots. -5. Use `line.stall_percent` to identify memory-bound or dependency-bound lines. -6. Drill down into `line.instructions` for instruction-level analysis. -7. Use relative paths for the target binary so the skill is portable. - -## Notes - -- Requires ROCm 7.0+ with `rocprofv3` support. -- Source mapping requires kernels compiled with `-g` (debug symbols). -- `source_lines` are automatically sorted by `total_cycles` (descending). -- Use `kernel_filter` to profile specific kernels by name (regex pattern). -- For Triton or other frameworks, ensure debug symbols are available in the compiled output. diff --git a/.agents/skills/metrix/SKILL.md b/.agents/skills/metrix/SKILL.md deleted file mode 100644 index 9a5564ece..000000000 --- a/.agents/skills/metrix/SKILL.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -name: metrix-profiling -description: Profile GPU kernels when performance analysis or optimization is required. Use for AMD ROCm GPU metrics, bandwidth, cache hit rates, coalescing, or kernel timing. ---- - -# Metrix: GPU Profiling - -Profile AMD GPU kernels and get human-readable metrics (bandwidth, cache, coalescing, FLOPS). Architecture is auto-detected. - -## When to Use - -- User asks to profile a GPU application or kernel -- Performance analysis, optimization, or bottleneck investigation -- Need HBM/L2/L1 bandwidth, hit rates, or compute metrics -- Need timing-only runs (fast, no hardware counters) - -## Instructions - -1. **Ensure the target runs on AMD ROCm** (e.g. `hipcc`-built binary or Python script that launches HIP/ROCm kernels). -2. **Choose execution path:** - - If a Metrix MCP server is available, use its profile tool with the same options below. - - Otherwise run the CLI or Python API from the environment where Metrix is installed. - -### CLI - -From the project or install prefix: - -```bash -# Profile with all metrics (auto-detected arch) -metrix ./my_app - -# Time only (fast, no counters) -metrix --time-only -n 10 ./my_app - -# Filter kernels by name -metrix --kernel matmul ./my_app - -# Specific metrics -metrix --metrics memory.l2_hit_rate,memory.coalescing_efficiency,compute.total_flops ./my_app - -# Save to JSON/CSV -metrix -o results.json ./my_app -``` - -Options: `--profile` (quick|memory|compute), `--metrics` (comma-separated), `--time-only`, `--kernel` (substring), `--num-replays`/`-n`, `--output`/`-o`, `--top K`, `--aggregate`, `--log`/`-l` (debug|info|warning|error), `--quiet`. - -### Python API - -```python -from metrix import Metrix - -profiler = Metrix() -results = profiler.profile("./my_app", num_replays=5) - -for kernel in results.kernels: - print(kernel.name, kernel.duration_us.avg) - for metric, stats in kernel.metrics.items(): - print(f" {metric}: {stats.avg}") -``` - -Use `metrics=[...]` for a subset; omit for all metrics. Use `cwd` when the binary expects a specific working directory. - -## Workflow - -1. Identify the executable or script to profile (e.g. `./app` or `python run_kernels.py`). -2. If only timing is needed, use `--time-only` for speed. -3. If full metrics are needed, run `metrix ./app` (or MCP equivalent); optionally restrict with `--kernel` or `--metrics`. -4. Interpret results: low L2 hit rate, low coalescing, or low HBM utilization suggest optimization targets. -5. For automation or tooling, use `-o results.json` and parse the JSON output. - -## Key Metrics (reference) - -- **Memory:** `memory.hbm_bandwidth_utilization`, `memory.l2_hit_rate`, `memory.l1_hit_rate`, `memory.coalescing_efficiency`, `memory.global_load_efficiency`, `memory.lds_bank_conflicts`, `memory.atomic_latency` -- **Compute:** `compute.total_flops`, `compute.hbm_gflops`, `compute.hbm_arithmetic_intensity`, `compute.l2_arithmetic_intensity`, `compute.l1_arithmetic_intensity` - -Use relative paths for the target binary and output files so the skill is portable across environments. diff --git a/.agents/skills/nexus/SKILL.md b/.agents/skills/nexus/SKILL.md deleted file mode 100644 index ad714bc4d..000000000 --- a/.agents/skills/nexus/SKILL.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -name: nexus-trace -description: Extract GPU kernel assembly and HIP source from HSA packet traces. Use when analyzing what code ran on the GPU, debugging kernel dispatch, or inspecting assembly and source mapping. ---- - -# Nexus: HSA Packet Source Code Extractor - -Intercepts HSA packets from a running process and extracts, per kernel, assembly and HIP source into a structured trace (e.g. JSON). Use for kernel-level inspection and assembly/source correlation. - -## When to Use - -- User needs to see which kernels ran and their assembly or HIP source -- Debugging or analyzing GPU dispatch and code generation -- Inspecting assembly-to-source mapping for a HIP (or ROCm) application - -## Instructions - -1. **Ensure the target runs on AMD ROCm** and uses HSA (e.g. HIP application or ROCm runtime). -2. **Choose execution path:** - - If a Nexus MCP server is available, use its tools: `list_kernels` to enumerate kernels in a trace, and `extract_kernel_code` to get assembly and HIP/source mapping (signature, files, lines). See `nexus/nexus/mcp/server.py` for tool parameters and schemas. - - Otherwise use the Python API from the environment where Nexus is installed. - -### Python API (recommended when no MCP) - -```python -from nexus import Nexus - -nexus = Nexus(log_level=1) -trace = nexus.run(["python", "my_gpu_script.py"]) - -# Or run a binary: -# trace = nexus.run(["./my_hip_app"]) - -for kernel in trace: - print(kernel.name, len(kernel.assembly), "instructions") - for i, asm_line in enumerate(kernel.assembly, 1): - print(f" {i}. {asm_line}") - for line_no, hip_line in zip(kernel.lines or range(1, len(kernel.hip)+1), kernel.hip): - print(f" {line_no}: {hip_line}") - -# Access by kernel name -k = trace["vector_add(float const*, float const*, float*, int)"] -print(k.assembly, k.hip, k.signature, k.files, k.lines) - -# Save/load trace -trace.save("trace.json") -loaded = Nexus.load("trace.json") -``` - -Set `log_level` (0–4) to control verbosity. Use relative paths for the run command and output file so the skill is portable. - -### Environment-based usage (no Python API) - -When the process cannot be launched via `nexus.run()`: - -1. Set `HSA_TOOLS_LIB` to the Nexus shared library path (e.g. `build/lib/libnexus.so` or the installed path). -2. Set `NEXUS_OUTPUT_FILE` to the output JSON path. -3. Set `NEXUS_LOG_LEVEL` (0–4) if needed. -4. Run the application as usual; it will be traced and the output file will contain the kernel data. - -Optional: `NEXUS_EXTRA_SEARCH_PREFIX` (colon-separated) for HIP source search; `TRITON_DISABLE_LINE_INFO=0` for Triton kernel line info. - -## Workflow - -1. Identify the command that runs the GPU workload (e.g. `python script.py` or `./app`). -2. If using the Python API: create `Nexus(log_level=...)`, call `nexus.run([...])`, then iterate `trace` and optionally `trace.save(...)`. -3. If using the env method: set `HSA_TOOLS_LIB` and `NEXUS_OUTPUT_FILE`, then run the app; open the JSON and parse the `kernels` structure. -4. Use kernel `signature`, `assembly`, `hip`, `files`, and `lines` to analyze what ran and map assembly back to source. -5. Use relative paths for commands and output files. - -## Notes - -- Nexus is intended for research/analysis; ensure the target environment has the Nexus library and compatible ROCm/HSA stack. -- For Triton kernels, enable line info via `TRITON_DISABLE_LINE_INFO=0` when using the Python API. diff --git a/.gitignore b/.gitignore index d8f9754f7..849669f1c 100644 --- a/.gitignore +++ b/.gitignore @@ -57,4 +57,6 @@ gpucore.* logs/ *.cap hsakmt_counters.csv -core \ No newline at end of file +core.intellikit/ +.agents/ +.rocprofv3/ diff --git a/examples/07_gemm_all_scatter/profiling/profile_with_metrix.py b/examples/07_gemm_all_scatter/profiling/profile_with_metrix.py new file mode 100644 index 000000000..41f693531 --- /dev/null +++ b/examples/07_gemm_all_scatter/profiling/profile_with_metrix.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +""" +Profile Example 07 (GEMM All-Scatter) with IntelliKit Metrix. + +Usage: + python profile_with_metrix.py [--output results.json] [--m M] [--n N] [--k K] + +This script profiles the persistent_gemm_all_scatter kernel using Metrix +and saves memory + compute metrics to a JSON file. +""" + +import argparse +import json +import os +import subprocess +import sys + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Profile GEMM All-Scatter with IntelliKit Metrix", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("-m", type=int, default=4096, help="Number of rows in matrix A") + parser.add_argument("-n", type=int, default=2048, help="Number of columns in matrix B") + parser.add_argument("-k", type=int, default=4096, help="Common dimension") + parser.add_argument("--num_ranks", type=int, default=2, help="Number of GPU ranks") + parser.add_argument("--num_replays", type=int, default=3, help="Number of profiling replays") + parser.add_argument("--master_port", type=int, default=29510, help="Torchrun master port") + parser.add_argument( + "--output", + type=str, + default="metrix_profiling_results.json", + help="Output JSON file for profiling results", + ) + return parser.parse_args() + + +def run_profile(profile_type, benchmark_cmd, output_file, num_replays, timeout=600): + """Run metrix profiling with the given profile type.""" + cmd = [ + "metrix", + "profile", + "--profile", + profile_type, + "--output", + output_file, + "--num-replays", + str(num_replays), + "--timeout", + str(timeout), + "--kernel", + "persistent_gemm", + benchmark_cmd, + ] + env = os.environ.copy() + env["HSA_NO_SCRATCH_RECLAIM"] = "1" + print(f"[profile_with_metrix] Running {profile_type} profile...") + result = subprocess.run(cmd, env=env, capture_output=True, text=True) + if result.returncode != 0: + print(f"[profile_with_metrix] WARNING: {profile_type} profiling failed (exit {result.returncode})") + print(result.stderr[-2000:] if result.stderr else "") + return None + return output_file + + +def extract_kernel_data(json_file, kernel_pattern="persistent_gemm"): + """Extract kernel metrics from a Metrix JSON output file.""" + if not json_file or not os.path.exists(json_file): + return {} + with open(json_file) as f: + data = json.load(f) + for key, value in data.items(): + if kernel_pattern in key: + return {key: value} + return {} + + +def main(): + args = parse_args() + + example_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + benchmark_script = os.path.join(example_dir, "benchmark.py") + + benchmark_cmd = ( + f"torchrun --nproc_per_node={args.num_ranks} --master_port={args.master_port} " + f"{benchmark_script} -m {args.m} -n {args.n} -k {args.k}" + ) + + # Run benchmark without profiling first to get timing + print("[profile_with_metrix] Running benchmark for timing...") + env = os.environ.copy() + env["HSA_NO_SCRATCH_RECLAIM"] = "1" + benchmark_result = subprocess.run( + benchmark_cmd.split() + ["--benchmark"], + env=env, + capture_output=True, + text=True, + ) + timing_data = {} + if benchmark_result.returncode == 0: + for line in benchmark_result.stdout.splitlines(): + if "tflops" in line.lower(): + print(f" {line.strip()}") + # Try to parse log.json if it was written + if os.path.exists("log.json"): + with open("log.json") as f: + timing_data = json.load(f) + else: + print("[profile_with_metrix] Benchmark run failed:", benchmark_result.stderr[-500:]) + + # Memory profile + mem_output = args.output.replace(".json", "_memory.json") + mem_file = run_profile("memory", benchmark_cmd, mem_output, args.num_replays) + mem_data = extract_kernel_data(mem_file) if mem_file else {} + + # Compute profile + compute_output = args.output.replace(".json", "_compute.json") + compute_file = run_profile("compute", benchmark_cmd, compute_output, args.num_replays) + compute_data = extract_kernel_data(compute_file) if compute_file else {} + + # Combine results + results = { + "benchmark": { + "description": "GEMM All-Scatter benchmark (Example 07)", + "matrix_size": {"M": args.m, "N": args.n, "K": args.k}, + "dtype": "fp16", + "world_size": args.num_ranks, + "algorithm": "persistent_gemm_all_scatter", + }, + "timing": timing_data, + "kernel_profiling": {}, + } + + # Merge memory and compute metrics per kernel + all_kernel_keys = set(list(mem_data.keys()) + list(compute_data.keys())) + for kernel_key in all_kernel_keys: + merged_metrics = {} + duration = None + if kernel_key in mem_data: + duration = mem_data[kernel_key].get("duration_us") + merged_metrics.update(mem_data[kernel_key].get("metrics", {})) + if kernel_key in compute_data: + if duration is None: + duration = compute_data[kernel_key].get("duration_us") + merged_metrics.update(compute_data[kernel_key].get("metrics", {})) + results["kernel_profiling"][kernel_key] = { + "duration_us": duration, + "metrics": merged_metrics, + } + + # Save combined results + with open(args.output, "w") as f: + json.dump(results, f, indent=2) + print(f"\n[profile_with_metrix] Results saved to: {args.output}") + + # Print summary + print("\n" + "=" * 70) + print("IntelliKit Metrix Profiling Summary: GEMM All-Scatter (Example 07)") + print("=" * 70) + print(f" Matrix: M={args.m}, N={args.n}, K={args.k} | fp16 | {args.num_ranks} GPUs") + if timing_data: + tflops = timing_data.get("tflops") + total_ms = timing_data.get("total_ms") + print(f" Throughput: {tflops:.2f} TFLOPS" if isinstance(tflops, float) else " Throughput: N/A") + print(f" Latency: {total_ms:.3f} ms" if isinstance(total_ms, float) else " Latency: N/A") + + for kernel_key, kdata in results["kernel_profiling"].items(): + kernel_name = kernel_key.split(":")[-1] if ":" in kernel_key else kernel_key + print(f"\nKernel: {kernel_name}") + dur = kdata.get("duration_us") + if dur: + print(f" Duration: avg={dur['avg']:.1f} µs, min={dur['min']:.1f} µs, max={dur['max']:.1f} µs") + for metric_name, stats in (kdata.get("metrics") or {}).items(): + avg = stats.get("avg", "N/A") + print(f" {metric_name}: {avg:.4f}" if isinstance(avg, float) else f" {metric_name}: {avg}") + print("=" * 70) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 7c6fe09d519c9f99d8d53c236fba204dfe5283b9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 12 Mar 2026 21:20:38 +0000 Subject: [PATCH 4/4] Add Metrix profiling results JSON and update .gitignore to track profiling results Co-authored-by: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com> --- .gitignore | 1 + .../profiling/metrix_profiling_results.json | 137 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 examples/07_gemm_all_scatter/profiling/metrix_profiling_results.json diff --git a/.gitignore b/.gitignore index 849669f1c..32d81d8a7 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ __pycache__/ !.devcontainer/devcontainer.json !.github/scripts/examples_config.json +!examples/**/profiling/*.json resources/ diff --git a/examples/07_gemm_all_scatter/profiling/metrix_profiling_results.json b/examples/07_gemm_all_scatter/profiling/metrix_profiling_results.json new file mode 100644 index 000000000..bb868ebaf --- /dev/null +++ b/examples/07_gemm_all_scatter/profiling/metrix_profiling_results.json @@ -0,0 +1,137 @@ +{ + "benchmark": { + "description": "GEMM All-Scatter benchmark (Example 07)", + "matrix_size": { + "M": 4096, + "N": 2048, + "K": 4096 + }, + "dtype": "fp16", + "world_size": 2, + "algorithm": "persistent_gemm_all_scatter", + "throughput_tflops": { + "small_4096x2048x4096": 146.5, + "default_8192x4608x36864": 611.8 + }, + "latency_ms": { + "small_4096x2048x4096": 0.469, + "default_8192x4608x36864": 4.549 + }, + "gpu": "MI300X", + "num_replays": 3, + "profiler": "IntelliKit Metrix" + }, + "kernel_profiling": { + "dispatch_452:persistent_gemm_all_scatter": { + "duration_us": { + "min": 300.953, + "max": 1176.495, + "avg": 602.9203333333334 + }, + "metrics": { + "memory.hbm_bandwidth_utilization": { + "min": 1.1002721046439674, + "max": 0.7477245561481676, + "avg": 0.8697097441859015, + "count": 3 + }, + "memory.hbm_read_bandwidth": { + "min": 51.344068336279484, + "max": 37.1381113065353, + "avg": 41.90058756030739, + "count": 3 + }, + "memory.hbm_write_bandwidth": { + "min": 6.9703532098507885, + "max": 2.4912901693175735, + "avg": 4.194028881545392, + "count": 3 + }, + "memory.bytes_transferred_hbm": { + "min": 141632512.0, + "max": 326411264.0, + "avg": 204862613.33333334, + "count": 3 + }, + "memory.l1_hit_rate": { + "min": 66.73415852187826, + "max": 66.73150012993763, + "avg": 66.7323862605845, + "count": 3 + }, + "memory.l2_hit_rate": { + "min": 58.868169864898746, + "max": 68.14033983835816, + "avg": 72.03185716187203, + "count": 3 + }, + "memory.l2_bandwidth": { + "min": 144.61376167942115, + "max": 123.2931168144759, + "avg": 164.37238361196927, + "count": 3 + }, + "memory.coalescing_efficiency": { + "min": 25.0, + "max": 25.0, + "avg": 25.0, + "count": 3 + }, + "memory.global_load_efficiency": { + "min": 12.538380920684359, + "max": 12.537379014048405, + "avg": 12.537712965135578, + "count": 3 + }, + "memory.global_store_efficiency": { + "min": 6.25, + "max": 6.25, + "avg": 6.25, + "count": 3 + }, + "memory.lds_bank_conflicts": { + "min": 0.0, + "max": 0.0, + "avg": 0.0, + "count": 3 + }, + "memory.atomic_latency": { + "min": 0.0, + "max": 0.0, + "avg": 0.0, + "count": 3 + }, + "compute.total_flops": { + "min": 34360786944.0, + "max": 720430284800.0, + "avg": 263050619562.66666, + "count": 3 + }, + "compute.hbm_gflops": { + "min": 116432.64425491252, + "max": 158677.29069695473, + "avg": 153753.77490063946, + "count": 3 + }, + "compute.hbm_arithmetic_intensity": { + "min": 240.72700288686437, + "max": 1972.8716381204908, + "avg": 1206.2224864348245, + "count": 3 + }, + "compute.l2_arithmetic_intensity": { + "min": 40.87954206519897, + "max": 856.680606365483, + "avg": 312.85160238506603, + "count": 3 + }, + "compute.l1_arithmetic_intensity": { + "min": 17.031704781704782, + "max": 357.09764000779626, + "avg": 130.38701652373527, + "count": 3 + } + } + } + } +}