diff --git a/.github/workflows/gemm-ci.yml b/.github/workflows/gemm-ci.yml index 0132e179..8db3f60f 100644 --- a/.github/workflows/gemm-ci.yml +++ b/.github/workflows/gemm-ci.yml @@ -95,3 +95,379 @@ jobs: else echo "PASS: 0 errors" fi + + gemm-interlaced: + name: GEMM interlaced [8x8 mesh] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake build-essential python3-pip + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install Python dependencies + run: | + pip install . + pip install torch --index-url https://download.pytorch.org/whl/cpu + + - name: Cache RISC-V toolchain + id: cache-riscv + uses: actions/cache@v4 + with: + path: ~/riscv + key: pulp-riscv-gcc-v1.0.16 + + - name: Install PULP RISC-V toolchain + if: steps.cache-riscv.outputs.cache-hit != 'true' + run: | + wget -q https://github.com/pulp-platform/pulp-riscv-gnu-toolchain/releases/download/v1.0.16/v1.0.16-pulp-riscv-gcc-ubuntu-18.tar.bz2 -O /tmp/riscv-toolchain.tar.bz2 + mkdir -p ~/riscv + tar -xjf /tmp/riscv-toolchain.tar.bz2 -C ~/riscv --strip-components=1 + rm /tmp/riscv-toolchain.tar.bz2 + + - name: Add toolchain to PATH + run: echo "$HOME/riscv/bin" >> "$GITHUB_PATH" + + - name: Verify toolchain + run: riscv32-unknown-elf-gcc --version + + - name: Cache GVSoC + id: cache-gvsoc + uses: actions/cache@v4 + with: + path: gvsoc + key: gvsoc-v1-tiles-8 + restore-keys: | + gvsoc-v1-tiles- + gvsoc-v1- + + - name: Clone GVSoC + if: steps.cache-gvsoc.outputs.cache-hit != 'true' + run: | + if [ ! -d "gvsoc" ]; then + git config --global url."https://github.com/".insteadOf "git@github.com:" + make gvsoc_init + fi + + - name: Install GVSoC Python dependencies + run: | + pip install -r gvsoc/core/requirements.txt + pip install -r gvsoc/requirements.txt + pip install -r gvsoc/gvrun/requirements.txt + + - name: Build GVSoC + run: make gvsoc tiles=8 + + - name: Generate golden data + run: bash scripts/gemm/gemm-gen.sh + + - name: Build GEMM interlaced test + run: TILES=8 COMPILER=GCC_PULP bash scripts/gemm/gemm-build.sh + + - name: Run GEMM interlaced test (8x8 mesh) + run: | + output=$(TILES=8 TEST_NAME=test_gemm_interlaced bash scripts/gemm/gemm-run.sh 2>&1) || true + result=$(echo "$output" | grep -E 'Errors:|Test complete' || true) + echo "$result" + errors=$(echo "$output" | grep -oP 'Errors:\s*\K[0-9]+' | head -1) + if [ -z "$errors" ]; then + echo "ERROR: Could not parse test output" + echo "Full output:" + echo "$output" + exit 1 + elif [ "$errors" -gt 0 ]; then + echo "FAIL: $errors errors" + exit 1 + else + echo "PASS: 0 errors" + fi + + gemm-l1-naive: + name: GEMM L1 naive [8x8 mesh] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake build-essential python3-pip + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install Python dependencies + run: | + pip install . + pip install torch --index-url https://download.pytorch.org/whl/cpu + + - name: Cache RISC-V toolchain + id: cache-riscv + uses: actions/cache@v4 + with: + path: ~/riscv + key: pulp-riscv-gcc-v1.0.16 + + - name: Install PULP RISC-V toolchain + if: steps.cache-riscv.outputs.cache-hit != 'true' + run: | + wget -q https://github.com/pulp-platform/pulp-riscv-gnu-toolchain/releases/download/v1.0.16/v1.0.16-pulp-riscv-gcc-ubuntu-18.tar.bz2 -O /tmp/riscv-toolchain.tar.bz2 + mkdir -p ~/riscv + tar -xjf /tmp/riscv-toolchain.tar.bz2 -C ~/riscv --strip-components=1 + rm /tmp/riscv-toolchain.tar.bz2 + + - name: Add toolchain to PATH + run: echo "$HOME/riscv/bin" >> "$GITHUB_PATH" + + - name: Verify toolchain + run: riscv32-unknown-elf-gcc --version + + - name: Cache GVSoC + id: cache-gvsoc + uses: actions/cache@v4 + with: + path: gvsoc + key: gvsoc-v1-tiles-8 + restore-keys: | + gvsoc-v1-tiles- + gvsoc-v1- + + - name: Clone GVSoC + if: steps.cache-gvsoc.outputs.cache-hit != 'true' + run: | + if [ ! -d "gvsoc" ]; then + git config --global url."https://github.com/".insteadOf "git@github.com:" + make gvsoc_init + fi + + - name: Install GVSoC Python dependencies + run: | + pip install -r gvsoc/core/requirements.txt + pip install -r gvsoc/requirements.txt + pip install -r gvsoc/gvrun/requirements.txt + + - name: Build GVSoC + run: make gvsoc tiles=8 + + - name: Generate golden data + run: bash scripts/gemm/gemm-gen.sh + + - name: Build GEMM L1 naive test + run: TILES=8 COMPILER=GCC_PULP bash scripts/gemm/gemm-build.sh + + - name: Run GEMM L1 naive test (8x8 mesh) + run: | + output=$(TILES=8 TEST_NAME=test_gemm_l1_naive bash scripts/gemm/gemm-run.sh 2>&1) || true + result=$(echo "$output" | grep -E 'Errors:|Test complete' || true) + echo "$result" + errors=$(echo "$output" | grep -oP 'Errors:\s*\K[0-9]+' | head -1) + if [ -z "$errors" ]; then + echo "ERROR: Could not parse test output" + echo "Full output:" + echo "$output" + exit 1 + elif [ "$errors" -gt 0 ]; then + echo "FAIL: $errors errors" + exit 1 + else + echo "PASS: 0 errors" + fi + + gemm-l1-interlaced: + name: GEMM L1 interlaced [8x8 mesh] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake build-essential python3-pip + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install Python dependencies + run: | + pip install . + pip install torch --index-url https://download.pytorch.org/whl/cpu + + - name: Cache RISC-V toolchain + id: cache-riscv + uses: actions/cache@v4 + with: + path: ~/riscv + key: pulp-riscv-gcc-v1.0.16 + + - name: Install PULP RISC-V toolchain + if: steps.cache-riscv.outputs.cache-hit != 'true' + run: | + wget -q https://github.com/pulp-platform/pulp-riscv-gnu-toolchain/releases/download/v1.0.16/v1.0.16-pulp-riscv-gcc-ubuntu-18.tar.bz2 -O /tmp/riscv-toolchain.tar.bz2 + mkdir -p ~/riscv + tar -xjf /tmp/riscv-toolchain.tar.bz2 -C ~/riscv --strip-components=1 + rm /tmp/riscv-toolchain.tar.bz2 + + - name: Add toolchain to PATH + run: echo "$HOME/riscv/bin" >> "$GITHUB_PATH" + + - name: Verify toolchain + run: riscv32-unknown-elf-gcc --version + + - name: Cache GVSoC + id: cache-gvsoc + uses: actions/cache@v4 + with: + path: gvsoc + key: gvsoc-v1-tiles-8 + restore-keys: | + gvsoc-v1-tiles- + gvsoc-v1- + + - name: Clone GVSoC + if: steps.cache-gvsoc.outputs.cache-hit != 'true' + run: | + if [ ! -d "gvsoc" ]; then + git config --global url."https://github.com/".insteadOf "git@github.com:" + make gvsoc_init + fi + + - name: Install GVSoC Python dependencies + run: | + pip install -r gvsoc/core/requirements.txt + pip install -r gvsoc/requirements.txt + pip install -r gvsoc/gvrun/requirements.txt + + - name: Build GVSoC + run: make gvsoc tiles=8 + + - name: Generate golden data + run: bash scripts/gemm/gemm-gen.sh + + - name: Build GEMM L1 interlaced test + run: TILES=8 COMPILER=GCC_PULP bash scripts/gemm/gemm-build.sh + + - name: Run GEMM L1 interlaced test (8x8 mesh) + run: | + output=$(TILES=8 TEST_NAME=test_gemm_l1_interlaced bash scripts/gemm/gemm-run.sh 2>&1) || true + result=$(echo "$output" | grep -E 'Errors:|Test complete' || true) + echo "$result" + errors=$(echo "$output" | grep -oP 'Errors:\s*\K[0-9]+' | head -1) + if [ -z "$errors" ]; then + echo "ERROR: Could not parse test output" + echo "Full output:" + echo "$output" + exit 1 + elif [ "$errors" -gt 0 ]; then + echo "FAIL: $errors errors" + exit 1 + else + echo "PASS: 0 errors" + fi + + gemm-fifo: + name: GEMM FIFO [8x8 mesh] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake build-essential python3-pip + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install Python dependencies + run: | + pip install . + pip install torch --index-url https://download.pytorch.org/whl/cpu + + - name: Cache RISC-V toolchain + id: cache-riscv + uses: actions/cache@v4 + with: + path: ~/riscv + key: pulp-riscv-gcc-v1.0.16 + + - name: Install PULP RISC-V toolchain + if: steps.cache-riscv.outputs.cache-hit != 'true' + run: | + wget -q https://github.com/pulp-platform/pulp-riscv-gnu-toolchain/releases/download/v1.0.16/v1.0.16-pulp-riscv-gcc-ubuntu-18.tar.bz2 -O /tmp/riscv-toolchain.tar.bz2 + mkdir -p ~/riscv + tar -xjf /tmp/riscv-toolchain.tar.bz2 -C ~/riscv --strip-components=1 + rm /tmp/riscv-toolchain.tar.bz2 + + - name: Add toolchain to PATH + run: echo "$HOME/riscv/bin" >> "$GITHUB_PATH" + + - name: Verify toolchain + run: riscv32-unknown-elf-gcc --version + + - name: Cache GVSoC + id: cache-gvsoc + uses: actions/cache@v4 + with: + path: gvsoc + key: gvsoc-v1-tiles-8 + restore-keys: | + gvsoc-v1-tiles- + gvsoc-v1- + + - name: Clone GVSoC + if: steps.cache-gvsoc.outputs.cache-hit != 'true' + run: | + if [ ! -d "gvsoc" ]; then + git config --global url."https://github.com/".insteadOf "git@github.com:" + make gvsoc_init + fi + + - name: Install GVSoC Python dependencies + run: | + pip install -r gvsoc/core/requirements.txt + pip install -r gvsoc/requirements.txt + pip install -r gvsoc/gvrun/requirements.txt + + - name: Build GVSoC + run: make gvsoc tiles=8 + + - name: Generate golden data + run: bash scripts/gemm/gemm-gen.sh + + - name: Build GEMM FIFO test + run: TILES=8 COMPILER=GCC_PULP bash scripts/gemm/gemm-build.sh + + - name: Run GEMM FIFO test (8x8 mesh) + run: | + output=$(TILES=8 TEST_NAME=test_gemm_fifo bash scripts/gemm/gemm-run.sh 2>&1) || true + result=$(echo "$output" | grep -E 'Errors:|Test complete' || true) + echo "$result" + errors=$(echo "$output" | grep -oP 'Errors:\s*\K[0-9]+' | head -1) + if [ -z "$errors" ]; then + echo "ERROR: Could not parse test output" + echo "Full output:" + echo "$output" + exit 1 + elif [ "$errors" -gt 0 ]; then + echo "FAIL: $errors errors" + exit 1 + else + echo "PASS: 0 errors" + fi diff --git a/.gitignore b/.gitignore index f4286112..64b59962 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,7 @@ regression_output* scripts/wave.do # Generated test inputs -tests/magia/mesh/gemm/*/include/* +tests/magia/mesh/gemm**/include/* # Editor .vscode/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77309f1c..ac833d16 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,7 +134,7 @@ magia_v2_tiles_1: .build: extends: [.base, .export_path] script: - - make clean build tiles=${tiles} target_platform=${target_platform} compiler=${compiler} + - make clean build tiles=${tiles} target_platform=${target_platform} compiler=${compiler} build_non_gitlab_tests=0 artifacts: paths: [ "build", "targets/${target_platform}/include/addr_map/tile_addr_map.h" ] diff --git a/Makefile b/Makefile index e7ec24f2..84f3c62a 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ profile_cmp ?= 0 profile_cmi ?= 0 profile_cmo ?= 0 profile_snc ?= 0 +build_non_gitlab_tests ?= 1 target_platform ?= magia_v2 compiler ?= GCC_PULP @@ -78,7 +79,7 @@ endif ifeq ($(compiler), LLVM) $(error COMING SOON!) endif - cmake -DTARGET_PLATFORM=$(target_platform) -DTILES=$(tiles) -DEVAL=$(eval) -DSTALLING=$(stalling) -DFSYNC_MM=$(fsync_mm) -DIDMA_MM=$(idma_mm) -DREDMULE_MM=$(redmule_mm) -DCOMPILER=$(compiler) -DPROFILE_CMP=$(profile_cmp) -DPROFILE_CMI=$(profile_cmi) -DPROFILE_CMO=$(profile_cmo) -DPROFILE_SNC=$(profile_snc) -B $(CMAKE_BUILDDIR) --trace-expand + cmake -DTARGET_PLATFORM=$(target_platform) -DTILES=$(tiles) -DEVAL=$(eval) -DSTALLING=$(stalling) -DFSYNC_MM=$(fsync_mm) -DIDMA_MM=$(idma_mm) -DREDMULE_MM=$(redmule_mm) -DCOMPILER=$(compiler) -DPROFILE_CMP=$(profile_cmp) -DPROFILE_CMI=$(profile_cmi) -DPROFILE_CMO=$(profile_cmo) -DPROFILE_SNC=$(profile_snc) -DBUILD_NON_GITLAB_TESTS=$(build_non_gitlab_tests) -B $(CMAKE_BUILDDIR) --trace-expand cmake --build $(CMAKE_BUILDDIR) --verbose set_mesh: diff --git a/scripts/gemm/gemm-gen.sh b/scripts/gemm/gemm-gen.sh index 4f5605ec..d8bdc823 100755 --- a/scripts/gemm/gemm-gen.sh +++ b/scripts/gemm/gemm-gen.sh @@ -19,7 +19,7 @@ DIM_E="${DIM_E:-64}" DIM_F="${DIM_F:-128}" SEED="${SEED:-42}" -python3 tests/magia/mesh/gemm/via_l2/gen_golden.py \ +python3 tests/magia/mesh/gemm_comm/gen_golden.py \ --dim-a "$DIM_A" --dim-b "$DIM_B" --dim-c "$DIM_C" \ --dim-d "$DIM_D" --dim-e "$DIM_E" --dim-f "$DIM_F" \ --seed "$SEED" diff --git a/scripts/gemm/gemm-run.sh b/scripts/gemm/gemm-run.sh index 507d6b89..adb9b00f 100755 --- a/scripts/gemm/gemm-run.sh +++ b/scripts/gemm/gemm-run.sh @@ -6,14 +6,19 @@ # TILES=2 GEMM_PLATFORM=rtl bash scripts/gemm/gemm-run.sh # # Environment variables (all optional): -# TILES mesh dimension N (default: 8) -# GEMM_PLATFORM gvsoc|rtl (default: gvsoc) -# MAKE make binary (default: make) +# TILES mesh dimension N (default: 8) +# GEMM_PLATFORM gvsoc|rtl (default: gvsoc) +# TEST_NAME GEMM variant binary name (default: test_gemm) +# e.g. test_gemm, test_gemm_interlaced, +# test_gemm_l1_naive, test_gemm_l1_interlaced, +# test_gemm_fifo +# MAKE make binary (default: make) set -euo pipefail TILES="${TILES:-8}" GEMM_PLATFORM="${GEMM_PLATFORM:-gvsoc}" +TEST_NAME="${TEST_NAME:-test_gemm}" MAKE="${MAKE:-make}" -"$MAKE" run test=test_gemm platform="$GEMM_PLATFORM" tiles="$TILES" +"$MAKE" run test="$TEST_NAME" platform="$GEMM_PLATFORM" tiles="$TILES" diff --git a/targets/magia_v2/include/utils/l1_fifo.h b/targets/magia_v2/include/utils/l1_fifo.h new file mode 100644 index 00000000..72684489 --- /dev/null +++ b/targets/magia_v2/include/utils/l1_fifo.h @@ -0,0 +1,220 @@ +/* + * L1 FIFO Communication Utils + * + * Lock-free per-tile FIFO mailbox in L1 memory. + * Any tile can push a message to any other tile's FIFO by writing to a + * pre-assigned slot. The owning tile polls slots for ready messages. + * + * Correctness relies on each (matrix_id, row_index) pair having exactly + * one producer, so each slot is written by at most one remote tile. + * A fence w,w before setting the valid flag ensures the consumer sees + * all payload data before the slot becomes visible. + * + * Memory layout at start of each tile's L1: + * +0x00: fifo_header_t (16 bytes) + * +0x10: slot array (num_slots * slot_stride bytes) + * + * Each slot: + * +0x00: valid (volatile uint32_t) — 0=empty, 1=ready + * +0x04: data_size (uint32_t) — actual payload size in bytes + * +0x08: matrix_id (uint32_t) — consumer-defined tag + * +0x0C: row_index (uint32_t) — global row position + * +0x10: payload data (slot_data_size bytes, inline) + */ + +#ifndef L1_FIFO_H +#define L1_FIFO_H + +#include "magia_tile_utils.h" +#include "addr_map/tile_addr_map.h" + +typedef struct { + uint32_t num_slots; + uint32_t slot_stride; + uint32_t scan_cursor; + uint32_t _pad; +} fifo_header_t; + +typedef struct { + volatile uint32_t valid; + uint32_t data_size; + uint32_t matrix_id; + uint32_t row_index; +} fifo_slot_t; + +#define FIFO_HEADER_SIZE (sizeof(fifo_header_t)) +#define FIFO_SLOT_META_SIZE (sizeof(fifo_slot_t)) + +/** + * Get pointer to a tile's FIFO header in L1. + */ +static inline fifo_header_t *fifo_get_header(uint32_t target_hartid) +{ + return (fifo_header_t *)(L1_BASE + target_hartid * L1_TILE_OFFSET); +} + +/** + * Get pointer to slot at given index. + */ +static inline fifo_slot_t *fifo_get_slot(fifo_header_t *hdr, uint32_t slot_idx) +{ + return (fifo_slot_t *)((uint8_t *)(hdr + 1) + slot_idx * hdr->slot_stride); +} + +/** + * Get pointer to a slot's inline payload data. + */ +static inline void *fifo_slot_data(fifo_slot_t *slot) +{ + return (void *)((uint8_t *)slot + FIFO_SLOT_META_SIZE); +} + +/** + * Initialize own tile's FIFO. Call once per tile at startup. + * + * num_slots: total number of message slots to pre-allocate + * slot_data_size: maximum payload size per slot in bytes + */ +static inline void fifo_init(uint32_t hartid, uint32_t num_slots, uint32_t slot_data_size) +{ + fifo_header_t *hdr = fifo_get_header(hartid); + uint32_t aligned_data_size = (slot_data_size + 3) & ~3u; + + hdr->num_slots = num_slots; + hdr->slot_stride = FIFO_SLOT_META_SIZE + aligned_data_size; + hdr->scan_cursor = 0; + hdr->_pad = 0; + + for (uint32_t i = 0; i < num_slots; i++) { + fifo_slot_t *slot = fifo_get_slot(hdr, i); + slot->valid = 0; + } + + asm volatile("fence w, w" ::: "memory"); +} + +/** + * Push data to a target tile's FIFO at a specific slot. + * + * The caller must provide a slot_idx that is unique per (matrix_id, row_index) + * pair for the target consumer. No locking is needed because each slot has + * exactly one producer. + */ +static inline void fifo_push(uint32_t target_hartid, + uint32_t slot_idx, + void *src_data, + uint32_t size_bytes, + uint32_t matrix_id, + uint32_t row_index) +{ + fifo_header_t *hdr = fifo_get_header(target_hartid); + fifo_slot_t *slot = fifo_get_slot(hdr, slot_idx); + void *dst = fifo_slot_data(slot); + + /* Copy payload word-by-word via volatile writes */ + volatile uint32_t *d = (volatile uint32_t *)dst; + + uint32_t *s = (uint32_t *)src_data; + uint32_t words = size_bytes / 4; + uint32_t remainder = size_bytes % 4; + + for (uint32_t i = 0; i < words; i++) + d[i] = s[i]; + + if (remainder) { + volatile uint8_t *db = (volatile uint8_t *)((uint8_t *)dst + words * 4); + uint8_t *sb = (uint8_t *)src_data + words * 4; + + for (uint32_t i = 0; i < remainder; i++) + db[i] = sb[i]; + } + + /* Write metadata */ + slot->data_size = size_bytes; + slot->matrix_id = matrix_id; + slot->row_index = row_index; + + /* Ensure all data and metadata writes are visible before the valid flag */ + asm volatile("fence w, w" ::: "memory"); + + /* Publish: makes the entry visible to the consumer */ + slot->valid = 1; +} + +/** + * Pop from own tile's FIFO. + * Returns 1 if a ready slot was found, 0 if all slots are empty. + * Scans round-robin from an internal cursor to avoid starvation. + */ +static inline uint32_t fifo_pop(uint32_t hartid, + uint32_t *out_data_ptr, + uint32_t *out_size, + uint32_t *out_matrix_id, + uint32_t *out_row_index) +{ + fifo_header_t *hdr = fifo_get_header(hartid); + uint32_t n = hdr->num_slots; + + if (n == 0) + return 0; + + uint32_t cursor = hdr->scan_cursor; + + for (uint32_t count = 0; count < n; count++) { + uint32_t idx = (cursor + count) % n; + fifo_slot_t *slot = fifo_get_slot(hdr, idx); + + if (slot->valid == 1) { + *out_data_ptr = (uint32_t)fifo_slot_data(slot); + *out_size = slot->data_size; + *out_matrix_id = slot->matrix_id; + *out_row_index = slot->row_index; + + /* Mark consumed */ + slot->valid = 0; + + /* Advance cursor past this slot */ + hdr->scan_cursor = (idx + 1) % n; + return 1; + } + } + + return 0; +} + +/** + * Publish a slot after its payload has been written externally (e.g. via DMA). + * Writes metadata, issues a fence, and sets the valid flag. + */ +static inline void fifo_slot_publish(uint32_t target_hartid, + uint32_t slot_idx, + uint32_t size_bytes, + uint32_t matrix_id, + uint32_t row_index) +{ + fifo_header_t *hdr = fifo_get_header(target_hartid); + fifo_slot_t *slot = fifo_get_slot(hdr, slot_idx); + + slot->data_size = size_bytes; + slot->matrix_id = matrix_id; + slot->row_index = row_index; + + asm volatile("fence w, w" ::: "memory"); + slot->valid = 1; +} + +/** + * Lock-free check: is the FIFO empty? + */ +static inline uint32_t fifo_is_empty(uint32_t hartid) +{ + fifo_header_t *hdr = fifo_get_header(hartid); + for (uint32_t i = 0; i < hdr->num_slots; i++) { + fifo_slot_t *slot = fifo_get_slot(hdr, i); + if (slot->valid == 1) + return 0; + } + return 1; +} + +#endif /* L1_FIFO_H */ diff --git a/targets/magia_v2/include/utils/performance_utils.h b/targets/magia_v2/include/utils/performance_utils.h index cb00079c..a667ce37 100644 --- a/targets/magia_v2/include/utils/performance_utils.h +++ b/targets/magia_v2/include/utils/performance_utils.h @@ -16,141 +16,166 @@ * * Authors: Victor Isachi * Alberto Dequino - * + * * MAGIA Utils */ #ifndef PERFORMANCE_UTILS_H #define PERFORMANCE_UTILS_H - /** * @brief Starts all performance counters */ -static inline void perf_start(void) { - // enable all counters - asm volatile("csrw 0x7E0, %0" :: "r"(0x1)); // Enable PCCR[0] - asm volatile("csrw 0x7E1, %0" :: "r"(0x1)); // Enable counting, , no saturation +static inline void perf_start(void) +{ + // GVSoC: mcycle count unconditionally; PULP PCER/PCMR (0x7E0/0x7E1) are unmodeled, so do not + // touch them here. + asm volatile("csrw 0xB00, %0" : : "r"(0)); // mcycle } /** * @brief Stops all performance counters */ -static inline void perf_stop(void) { - asm volatile("csrw 0x320, %0" : : "r"(0xffffffff)); +static inline void perf_stop(void) +{ + asm volatile("csrw 0x320, %0" : : "r"(0xffffffff)); // mcountinhibit: stop standard counters } /** * @brief Resets all performance counters to 0 without stopping them */ -static inline void perf_reset(void) { - asm volatile("csrw 0x780, %0" : : "r"(0)); +static inline void perf_reset(void) +{ + asm volatile("csrw 0xB00, %0" : : "r"(0)); } /** * @brief Returns the cycles of the performance counter */ -static inline unsigned int perf_get_cycles(){ +static inline unsigned int perf_get_cycles() +{ unsigned int value = 0; - asm volatile("csrr %0, 0x780" : "=r"(value)); + asm volatile("csrr %0, 0xB00" : "=r"(value)); return value; } /** * @brief Returns the n. instructions of the performance counter */ -static inline unsigned int perf_get_instr(){ +static inline unsigned int perf_get_instr() +{ unsigned int value = 0; - asm volatile ("csrr %0, 0xB02" : "=r" (value)); + asm volatile("csrr %0, 0xB02" : "=r"(value)); return value; } -static inline void sentinel_start(){ +static inline void sentinel_start() +{ asm volatile("addi x0, x0, 0x5AA" ::); } -static inline void sentinel_end(){ +static inline void sentinel_end() +{ asm volatile("addi x0, x0, 0x5FF" ::); } // Start input communication sentinel accumulator -static inline void stnl_cmi_s(){ +static inline void stnl_cmi_s() +{ asm volatile("addi x0, x0, 0x50B" ::); } // Start output communication sentinel accumulator -static inline void stnl_cmo_s(){ +static inline void stnl_cmo_s() +{ asm volatile("addi x0, x0, 0x51B" ::); } // Start computation sentinel accumulator -static inline void stnl_cmp_s(){ +static inline void stnl_cmp_s() +{ asm volatile("addi x0, x0, 0x52B" ::); } // Start synchronization sentinel accumulator -static inline void stnl_snc_s(){ +static inline void stnl_snc_s() +{ asm volatile("addi x0, x0, 0x53B" ::); } // Start timeslot sentinel accumulator -static inline void stnl_ts_s(){ +static inline void stnl_ts_s() +{ asm volatile("addi x0, x0, 0x5FB" ::); } // Finish (record) input communication sentinel accumulator -static inline void stnl_cmi_f(){ +static inline void stnl_cmi_f() +{ asm volatile("addi x0, x0, 0x50C" ::); } // Finish (record) output communication sentinel accumulator -static inline void stnl_cmo_f(){ +static inline void stnl_cmo_f() +{ asm volatile("addi x0, x0, 0x51C" ::); } // Finish (record) computation sentinel accumulator -static inline void stnl_cmp_f(){ +static inline void stnl_cmp_f() +{ asm volatile("addi x0, x0, 0x52C" ::); } // Finish (record) synchronization sentinel accumulator -static inline void stnl_snc_f(){ +static inline void stnl_snc_f() +{ asm volatile("addi x0, x0, 0x53C" ::); } // Finish (record) timeslot sentinel accumulator -static inline void stnl_ts_f(){ +static inline void stnl_ts_f() +{ asm volatile("addi x0, x0, 0x5FC" ::); } // Report input communication sentinel accumulator -static inline void stnl_cmi_r(){ +static inline void stnl_cmi_r() +{ asm volatile("addi x0, x0, 0x50D" ::); } // Report output communication sentinel accumulator -static inline void stnl_cmo_r(){ +static inline void stnl_cmo_r() +{ asm volatile("addi x0, x0, 0x51D" ::); } // Report computation sentinel accumulator -static inline void stnl_cmp_r(){ +static inline void stnl_cmp_r() +{ asm volatile("addi x0, x0, 0x52D" ::); } // Report synchronization sentinel accumulator -static inline void stnl_snc_r(){ +static inline void stnl_snc_r() +{ asm volatile("addi x0, x0, 0x53D" ::); } // Report timeslot sentinel accumulator -static inline void stnl_ts_r(){ +static inline void stnl_ts_r() +{ asm volatile("addi x0, x0, 0x5FD" ::); } // Report global input communication, output communication and computation overheads -static inline void stnl_r(){ +static inline void stnl_r() +{ asm volatile("addi x0, x0, 0x5EE" ::); } -#endif \ No newline at end of file +/* Accumulate elapsed cycles since t0 into acc. */ +#define PERF_DELTA(acc, t0) ((acc) += perf_get_cycles() - (t0)) + +#endif diff --git a/targets/magia_v2/src/io.c b/targets/magia_v2/src/io.c index a86517a8..01bce787 100644 --- a/targets/magia_v2/src/io.c +++ b/targets/magia_v2/src/io.c @@ -25,6 +25,7 @@ #include #include "addr_map/tile_addr_map.h" +__attribute__((optimize("no-tree-loop-distribute-patterns"))) void *memset(void *m, int c, size_t n) { char *s = (char *)m; diff --git a/tests/magia/mesh/CMakeLists.txt b/tests/magia/mesh/CMakeLists.txt index b3193731..9dc7a476 100644 --- a/tests/magia/mesh/CMakeLists.txt +++ b/tests/magia/mesh/CMakeLists.txt @@ -14,8 +14,13 @@ add_subdirectory(mm_os) add_subdirectory(idma_2d) add_subdirectory(idma_1d) #add_subdirectory(flatatt) -if(TILES STREQUAL "8") - add_subdirectory(gemm/via_l2) +option(BUILD_NON_GITLAB_TESTS "Build non GitLab tests" ON) +if(BUILD_NON_GITLAB_TESTS) + add_subdirectory(gemm_comm/via_L2/naive) + add_subdirectory(gemm_comm/via_L2/interlaced) + add_subdirectory(gemm_comm/via_L1/naive) + add_subdirectory(gemm_comm/via_L1/interlaced) + add_subdirectory(gemm_comm/fifo) endif() # add_subdirectory(float) add_subdirectory(cemm_global) diff --git a/tests/magia/mesh/gemm_comm/README.md b/tests/magia/mesh/gemm_comm/README.md new file mode 100644 index 00000000..eabe31cb --- /dev/null +++ b/tests/magia/mesh/gemm_comm/README.md @@ -0,0 +1,43 @@ +# GEMM Communication Tests + +![GEMM communication test](gemm_comm_test.png) + +A 4-GEMM chain run across the mesh, exploring different inter-tile communication strategies. All variants share the same golden data generated by `gen_golden.py`. + +## Available tests + +| Variant directory | Test name (`make run test=...`) | +| ---------------------- | ------------------------------- | +| `via_L2/naive/` | `test_gemm_l2_naive` | +| `via_L2/interlaced/` | `test_gemm_l2_interlaced` | +| `via_L1/naive/` | `test_gemm_l1_naive` | +| `via_L1/interlaced/` | `test_gemm_l1_interlaced` | +| `fifo/` | `test_gemm_fifo` | + +## Running + +```bash +# Build (requires test.h golden data — see gen_golden.py) +make clean build target_platform=magia_v2 tiles=2 compiler=GCC_PULP + +# GVSoC +make run platform=gvsoc test=test_gemm_l2_naive tiles=2 +make run platform=gvsoc test=test_gemm_l2_interlaced tiles=2 +make run platform=gvsoc test=test_gemm_l1_naive tiles=2 +make run platform=gvsoc test=test_gemm_l1_interlaced tiles=2 +make run platform=gvsoc test=test_gemm_fifo tiles=2 + +# RTL (requires the MAGIA RTL repo + Modelsim, see CLAUDE.md) +make run platform=rtl test=test_gemm_l2_naive tiles=2 +# ...etc +``` + +## Variants at a glance + +- **`via_L2/naive`** — intermediate results round-trip through L2; global barriers between phases. +- **`via_L2/interlaced`** — same chain, but GEMM4 prefetches `M5` during Phase 1 and skips the `M5` DMA in Phase 3. +- **`via_L1/naive`** — intermediate results DMA'd L1→L1 directly (only I/O touches L2). +- **`via_L1/interlaced`** — L1-direct variant with prefetching, like `via_L2/interlaced` but over L1. +- **`fifo`** — uses `l1_fifo.h` (lock-free slot-based mailbox) for inter-tile communication, eliminating global barriers between phases. + +See `CLAUDE.md` ("GEMM Chain Pipeline") for the full design notes. diff --git a/tests/magia/mesh/gemm_comm/fifo/CMakeLists.txt b/tests/magia/mesh/gemm_comm/fifo/CMakeLists.txt new file mode 100644 index 00000000..47a535b2 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/fifo/CMakeLists.txt @@ -0,0 +1,16 @@ +set(TEST_NAME test_gemm_fifo) + +set(FIFO_BATCH_ROWS "1" CACHE STRING "Rows per FIFO push (default 1)") + +file(GLOB_RECURSE TEST_SRCS "src/*.c") +list(FILTER TEST_SRCS EXCLUDE REGEX ".*/test_with_prints\\.c$") + +add_executable(${TEST_NAME} ${TEST_SRCS}) +target_include_directories(${TEST_NAME} PUBLIC include ${CMAKE_CURRENT_SOURCE_DIR}/..) +target_compile_options(${TEST_NAME} PRIVATE -O2) +target_compile_definitions(${TEST_NAME} PRIVATE FIFO_BATCH_ROWS=${FIFO_BATCH_ROWS}) +target_link_libraries(${TEST_NAME} PUBLIC runtime hal) + +add_custom_command( + TARGET ${TEST_NAME} POST_BUILD + COMMAND ${CMAKE_OBJDUMP} -dhS -Mmarch=${ISA} $ > $.s) diff --git a/tests/magia/mesh/gemm_comm/fifo/src/test.c b/tests/magia/mesh/gemm_comm/fifo/src/test.c new file mode 100644 index 00000000..34f69518 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/fifo/src/test.c @@ -0,0 +1,947 @@ +#include +#include "test.h" + +#include "tile.h" +#include "gemm_utils.h" +#include "fsync.h" +#include "idma.h" +#include "redmule.h" +#include "eventunit.h" +#include "utils/l1_fifo.h" + +#define WAIT_MODE WFE + +/** + * Tile group definitions for 8x8 mesh (64 tiles). + * + * GEMM1 tiles: [0, 1, 8, 9] (4 tiles, top-left 2x2 block) + * GEMM2 tiles: [16-19, 24-27, 32-35, + * 40-43, 48-51, 56-59] (24 tiles, rows 2-7 cols 0-3) + * GEMM3 tiles: [2-7, 10-15] (12 tiles, rows 0-1 cols 2-7) + * GEMM4 tiles: [20-23, 28-31, 36-39, + * 44-47, 52-55, 60-63] (24 tiles, rows 2-7 cols 4-7) + */ +#define GEMM1_N_TILES 4 +#define GEMM2_N_TILES 24 +#define GEMM3_N_TILES 12 +#define GEMM4_N_TILES 24 + +#define abs_threshold_millis 8 /* 0.008 expressed as integer millis */ + +/* FIFO message type tags */ +#define MATRIX_R1 0u +#define MATRIX_R2 1u +#define MATRIX_R3 2u + +/* + * 16 KB reserved at the start of each tile's L1 for the FIFO header, + * linked-list nodes, and payload data. Workspace buffers follow after. + */ +#define FIFO_RESERVE_SIZE 0x4000u + +/* + * Fraction of a tile's rows to push per fifo_push_dma() call. + * Set at build time via -DFIFO_BATCH_FRAC=F (default 0.2, i.e. 5 steps of 20%). + * If <= 0, falls back to one row at a time. + */ +#ifndef FIFO_BATCH_FRAC +#define FIFO_BATCH_FRAC 0.2f +#endif + +/* + * Compute the batch size (rows per push) for a tile with `total` rows. + * Returns ceil(total * frac), minimum 1. + * If frac <= 0, returns 1 (one row at a time). + */ +static uint32_t compute_batch(uint32_t total, float frac) +{ + if (frac <= 0.0f || total == 0) + return 1; + uint32_t b = (uint32_t)(total * frac + 0.9999f); /* ceiling */ + return (b < 1) ? 1 : b; +} + +/* + * Compute chunk i out of n_chunks for splitting `total` row count. + * Distributes the remainder across the first chunks (mirrors get_row_range). + */ +static void +compute_chunk(uint32_t total, uint32_t n_chunks, uint32_t idx, uint32_t *start, uint32_t *len) +{ + uint32_t base = total / n_chunks; + uint32_t rem = total % n_chunks; + *start = idx * base + (idx < rem ? idx : rem); + *len = base + (idx < rem ? 1 : 0); +} + +static const uint32_t gemm1_tiles[GEMM1_N_TILES] = {0, 1, 8, 9}; + +static const uint32_t gemm2_tiles[GEMM2_N_TILES] = {16, 17, 18, 19, 24, 25, 26, 27, 32, 33, 34, 35, + 40, 41, 42, 43, 48, 49, 50, 51, 56, 57, 58, 59}; + +static const uint32_t gemm3_tiles[GEMM3_N_TILES] = {2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15}; + +static const uint32_t gemm4_tiles[GEMM4_N_TILES] = {20, 21, 22, 23, 28, 29, 30, 31, 36, 37, 38, 39, + 44, 45, 46, 47, 52, 53, 54, 55, 60, 61, 62, 63}; + +static int get_local_idx(uint32_t hartid, const uint32_t *tiles, uint32_t n_tiles) +{ + for (uint32_t i = 0; i < n_tiles; i++) + if (tiles[i] == hartid) + return (int)i; + return -1; +} + +static void get_row_range(uint32_t local_idx, + uint32_t n_tiles, + uint32_t total_rows, + uint32_t *start_row, + uint32_t *num_rows) +{ + uint32_t base = total_rows / n_tiles; + uint32_t rem = total_rows % n_tiles; + *start_row = local_idx * base + (local_idx < rem ? local_idx : rem); + *num_rows = base + (local_idx < rem ? 1 : 0); +} + +static void mem_set_zero(uint32_t o, uint32_t dim) +{ + for (uint32_t i = 0; i < dim; i++) + mmio16(o + i * 2) = 0x0000; +} + +/* + * Push data to a target tile's FIFO slot using iDMA for the payload transfer. + */ +static void fifo_push_dma(uint32_t target_hartid, + uint32_t slot_idx, + uint32_t src_addr, + uint32_t size_bytes, + uint32_t matrix_id, + uint32_t row_index, + idma_controller_t *idma_ctrl, + eu_controller_t *eu_ctrl, + uint32_t *cyc_push) +{ + fifo_header_t *hdr = fifo_get_header(target_hartid); + fifo_slot_t *slot = fifo_get_slot(hdr, slot_idx); + uint32_t dst = (uint32_t)fifo_slot_data(slot); + + uint32_t t0 = perf_get_cycles(); + /* DMA payload: local L1 (OBI) → remote L1 (AXI) */ + idma_memcpy_1d(idma_ctrl, 1, dst, src_addr, size_bytes); + eu_idma_wait_o2a(eu_ctrl, WAIT_MODE); + + /* Publish metadata + valid flag */ + fifo_slot_publish(target_hartid, slot_idx, size_bytes, matrix_id, row_index); + PERF_DELTA(*cyc_push, t0); +} + +/* + * Push completed R3 rows to all overlapping GEMM4 tile FIFOs. + */ +static void push_r3_to_gemm4(uint32_t global_row, + uint32_t batch_rows, + uint32_t obi_r3_batch, + idma_controller_t *idma_ctrl, + eu_controller_t *eu_ctrl, + uint32_t *cyc_push) +{ + for (uint32_t k = 0; k < GEMM4_N_TILES; k++) { + uint32_t g4_start, g4_nrows; + get_row_range(k, GEMM4_N_TILES, DIM_A, &g4_start, &g4_nrows); + if (g4_nrows == 0) + continue; + + uint32_t ov_start = global_row > g4_start ? global_row : g4_start; + uint32_t ov_end = (global_row + batch_rows) < (g4_start + g4_nrows) + ? (global_row + batch_rows) + : (g4_start + g4_nrows); + if (ov_start >= ov_end) + continue; + + uint32_t ov_count = ov_end - ov_start; + uint32_t src_off = (ov_start - global_row) * DIM_E * 2; + uint32_t slot_idx = ov_start - g4_start; + fifo_push_dma(gemm4_tiles[k], + slot_idx, + obi_r3_batch + src_off, + ov_count * DIM_E * 2, + MATRIX_R3, + ov_start, + idma_ctrl, + eu_ctrl, + cyc_push); + } +} + +/* + * Accumulate a partial R3 contribution for R1 batch at local row lr, + * using R2 rows [k_start .. k_start + k_len). + * + * Returns the number of newly completed R3 rows (pushed to GEMM4), or 0. + */ +static uint32_t gemm3_partial_accum(uint32_t lr, + uint32_t k_start, + uint32_t k_len, + uint32_t r1_batch, + uint32_t r1_ptr, + uint32_t obi_r2, + uint32_t obi_r3, + uint32_t obi_r1_tmp, + uint32_t start_row, + uint32_t *r3_k_done, + uint8_t *r3_pushed, + redmule_controller_t *redmule_ctrl, + eu_controller_t *eu_ctrl, + idma_controller_t *idma_ctrl, + uint32_t *cyc_compute, + uint32_t *cyc_push) +{ + uint32_t A_ptr; + + if (r1_batch == 1) { + /* Single row: R1[0, k_start:k_start+k_len] is contiguous within the row */ + A_ptr = r1_ptr + k_start * 2; + } else { + /* Multiple rows: extract columns k_start..k_start+k_len into temp buffer */ + for (uint32_t r = 0; r < r1_batch; r++) { + volatile uint16_t *src = (volatile uint16_t *)(r1_ptr + (r * DIM_C + k_start) * 2); + volatile uint16_t *dst = (volatile uint16_t *)(obi_r1_tmp + r * k_len * 2); + for (uint32_t c = 0; c < k_len; c++) + dst[c] = src[c]; + } + A_ptr = obi_r1_tmp; + } + + uint32_t B_ptr = obi_r2 + k_start * DIM_E * 2; + uint32_t obi_r3_batch = obi_r3 + lr * DIM_E * 2; + + uint32_t t0 = perf_get_cycles(); + redmule_gemm(redmule_ctrl, + A_ptr, + B_ptr, + obi_r3_batch, + (uint16_t)r1_batch, + (uint16_t)k_len, + (uint16_t)DIM_E); + eu_redmule_wait(eu_ctrl, WAIT_MODE); + PERF_DELTA(*cyc_compute, t0); + + r3_k_done[lr] += k_len; + + if (r3_k_done[lr] == DIM_C && !r3_pushed[lr]) { + r3_pushed[lr] = 1; + uint32_t global_row = start_row + lr; + push_r3_to_gemm4(global_row, r1_batch, obi_r3_batch, idma_ctrl, eu_ctrl, cyc_push); + return r1_batch; + } + + return 0; +} + +int main(void) +{ + /* 0. Initialization */ + uint32_t hartid = get_hartid(); + uint32_t l1_tile_base = get_l1_base(hartid); + + /* Init iDMA */ + idma_config_t idma_cfg = {.hartid = hartid}; + idma_controller_t idma_ctrl = { + .base = NULL, + .cfg = &idma_cfg, + .api = &idma_api, + }; + idma_init(&idma_ctrl); + + /* Init RedMulE */ + redmule_config_t redmule_cfg = {.hartid = hartid}; + redmule_controller_t redmule_ctrl = { + .base = NULL, + .cfg = &redmule_cfg, + .api = &redmule_api, + }; + redmule_init(&redmule_ctrl); + + /* Init FractalSync */ + fsync_config_t fsync_cfg = {.hartid = hartid}; + fsync_controller_t fsync_ctrl = { + .base = NULL, + .cfg = &fsync_cfg, + .api = &fsync_api, + }; + fsync_init(&fsync_ctrl); + +/* Init Event Unit */ +#if STALLING == 0 + eu_config_t eu_cfg = {.hartid = hartid}; + eu_controller_t eu_ctrl = { + .base = NULL, + .cfg = &eu_cfg, + .api = &eu_api, + }; + + eu_init(&eu_ctrl); + eu_clear_events(0xFFFFFFFF); + eu_fsync_init(&eu_ctrl, 0); + eu_idma_init(&eu_ctrl, 0); + eu_redmule_init(&eu_ctrl, 0); +#endif + + /* Per-tile cycle-count breakdown (accumulated across phases) */ + uint32_t cyc_barrier = 0; /* fsync barrier wait */ + uint32_t cyc_preamble = 0; /* GEMM1 preamble (setup before first load), excluding prints */ + uint32_t cyc_cmi = 0; /* input DMA: L2→L1 matrix loads + GEMM3 R2 workspace copy */ + uint32_t cyc_compute = 0; /* RedMulE GEMM time */ + uint32_t cyc_push = 0; /* fifo_push_dma: DMA + publish */ + uint32_t cyc_spin = 0; /* fifo_pop spin (consumer idle) */ + uint32_t cyc_cmo = 0; /* output DMA: L1→L2 (GEMM4 only) */ + uint32_t _t0; + + /* Determine group membership for this tile (needed to size the FIFO) */ + int gemm1_idx = get_local_idx(hartid, gemm1_tiles, GEMM1_N_TILES); + int gemm2_idx = get_local_idx(hartid, gemm2_tiles, GEMM2_N_TILES); + int gemm3_idx = get_local_idx(hartid, gemm3_tiles, GEMM3_N_TILES); + int gemm4_idx = get_local_idx(hartid, gemm4_tiles, GEMM4_N_TILES); + + /* + * Initialize this tile's FIFO before the startup barrier. + * Slot count and payload size depend on the tile's consumer role. + * Only needed for GEMM 3 and GEMM 4 (which have as input the result + * of other GEMMs, performed by other tiles). + */ + uint32_t num_slots = 0; + uint32_t slot_data_size = 0; + + if (gemm3_idx >= 0) { + uint32_t g3s, g3n; + + get_row_range((uint32_t)gemm3_idx, GEMM3_N_TILES, DIM_A, &g3s, &g3n); + + uint32_t r1_batch = compute_batch(g3n, FIFO_BATCH_FRAC); + uint32_t r2_batch = compute_batch(DIM_C, FIFO_BATCH_FRAC); + + uint32_t r1_payload = r1_batch * DIM_C * 2; + uint32_t r2_payload = r2_batch * DIM_E * 2; + + num_slots = g3n + DIM_C; + slot_data_size = r1_payload > r2_payload ? r1_payload : r2_payload; + } else if (gemm4_idx >= 0) { + uint32_t g4s, g4n; + + get_row_range((uint32_t)gemm4_idx, GEMM4_N_TILES, DIM_A, &g4s, &g4n); + + uint32_t r3_batch = compute_batch(g4n, FIFO_BATCH_FRAC); + + num_slots = g4n; + slot_data_size = r3_batch * DIM_E * 2; + } + + fifo_init(hartid, num_slots, slot_data_size); + + /* + * Startup barrier: ensure all tiles finish crt0.S BSS zeroing (which + * touches global output buffers) and FIFO initialization before any + * tile begins pushing into another tile's FIFO. + */ + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + perf_reset(); + perf_start(); + + /* + * L1 workspace starts at FIFO_RESERVE_SIZE bytes past the tile base. + * The first FIFO_RESERVE_SIZE bytes are reserved for the FIFO header + * and slot array written by remote pushers. + */ + uint32_t ws = l1_tile_base + FIFO_RESERVE_SIZE; + + _t0 = perf_get_cycles(); + + /* ------------------------------------------------------------------ */ + /* GEMM1: R1 = M1 @ M2, push R1 rows to GEMM3 FIFOs */ + /* */ + /* Double-buffered: split the local M1 row slice into chunks of */ + /* compute_batch(num_rows, FIFO_BATCH_FRAC) rows. While RedMulE */ + /* computes chunk i, iDMA prefetches chunk i+1 into the alternate */ + /* ping-pong buffer. M2 is loaded once and shared across all chunks. */ + /* ------------------------------------------------------------------ */ + if (gemm1_idx >= 0) { + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + + uint32_t start_row, num_rows; + get_row_range((uint32_t)gemm1_idx, GEMM1_N_TILES, DIM_A, &start_row, &num_rows); + + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + + if (num_rows > 0) { + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + + uint32_t chunk_max = compute_batch(num_rows, FIFO_BATCH_FRAC); + uint32_t n_chunks = (num_rows + chunk_max - 1) / chunk_max; + + /* Workspace: [M2 | M1_pp[0] | M1_pp[1] | R1_pp[0] | R1_pp[1]] */ + uint32_t obi_m2 = ws; + uint32_t obi_m1[2] = {obi_m2 + DIM_B * DIM_C * 2, + obi_m2 + DIM_B * DIM_C * 2 + chunk_max * DIM_B * 2}; + uint32_t obi_r1[2] = {obi_m1[1] + chunk_max * DIM_B * 2, + obi_m1[1] + chunk_max * DIM_B * 2 + chunk_max * DIM_C * 2}; + + PERF_DELTA(cyc_preamble, _t0); + + /* Load full M2 from L2 */ + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m2_inp, obi_m2, DIM_B * DIM_C * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + + /* Prime the pipeline: load M1_pp[0] */ + uint32_t c0s, c0l; + compute_chunk(num_rows, n_chunks, 0, &c0s, &c0l); + + uint32_t cyc_cmi_pre_prime = cyc_cmi; + + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m1_inp + (start_row + c0s) * DIM_B * 2, + obi_m1[0], + c0l * DIM_B * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + + for (uint32_t i = 0; i < n_chunks; i++) { + uint32_t cs, cl; + compute_chunk(num_rows, n_chunks, i, &cs, &cl); + uint32_t pp = i & 1u; + uint32_t has_next = (i + 1u < n_chunks); + + /* Kick off chunk i+1 load (overlaps with chunk i compute) */ + if (has_next) { + uint32_t ns, nl; + compute_chunk(num_rows, n_chunks, i + 1, &ns, &nl); + uint32_t cyc_cmi_before = cyc_cmi; + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m1_inp + (start_row + ns) * DIM_B * 2, + obi_m1[(i + 1) & 1u], + nl * DIM_B * 2); + PERF_DELTA(cyc_cmi, _t0); + } + + /* Compute R1 chunk i (zero-init + GEMM issue + wait) */ + uint32_t cyc_compute_before = cyc_compute; + _t0 = perf_get_cycles(); + mem_set_zero(obi_r1[pp], cl * DIM_C); + redmule_gemm(&redmule_ctrl, + obi_m1[pp], + obi_m2, + obi_r1[pp], + (uint16_t)cl, + (uint16_t)DIM_B, + (uint16_t)DIM_C); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_compute, _t0); + + if (has_next) { + uint32_t cyc_cmi_wait_before = cyc_cmi; + _t0 = perf_get_cycles(); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + } + + /* + * Push this chunk's R1 rows to each GEMM3 tile that owns + * overlapping rows. Producer-side window for the chunk is + * [chunk_g_start, chunk_g_end). + */ + uint32_t chunk_g_start = start_row + cs; + uint32_t chunk_g_end = chunk_g_start + cl; + for (uint32_t j = 0; j < GEMM3_N_TILES; j++) { + uint32_t g3_start, g3_nrows; + get_row_range(j, GEMM3_N_TILES, DIM_A, &g3_start, &g3_nrows); + if (g3_nrows == 0) + continue; + + uint32_t ov_start = chunk_g_start > g3_start ? chunk_g_start : g3_start; + uint32_t ov_end = + chunk_g_end < (g3_start + g3_nrows) ? chunk_g_end : (g3_start + g3_nrows); + if (ov_start >= ov_end) + continue; + + uint32_t g3_r1_batch = compute_batch(g3_nrows, FIFO_BATCH_FRAC); + for (uint32_t r = ov_start; r < ov_end; r += g3_r1_batch) { + uint32_t batch = (r + g3_r1_batch <= ov_end) ? g3_r1_batch : (ov_end - r); + uint32_t src_off = (r - chunk_g_start) * DIM_C * 2; + uint32_t slot_idx = r - g3_start; + + uint32_t cyc_push_before = cyc_push; + fifo_push_dma(gemm3_tiles[j], + slot_idx, + obi_r1[pp] + src_off, + batch * DIM_C * 2, + MATRIX_R1, + r, + &idma_ctrl, + &eu_ctrl, + &cyc_push); + } + } + } + } + } + + /* ------------------------------------------------------------------ */ + /* GEMM2: R2 = M3 @ M4, push R2 rows to all GEMM3 FIFOs */ + /* */ + /* Double-buffered using the same chunk strategy as GEMM1. M4 is */ + /* loaded once; M3 chunks are ping-pong'd against RedMulE compute. */ + /* ------------------------------------------------------------------ */ + if (gemm2_idx >= 0) { + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + + uint32_t start_row, num_rows; + get_row_range((uint32_t)gemm2_idx, GEMM2_N_TILES, DIM_C, &start_row, &num_rows); + + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + + if (num_rows > 0) { + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + + uint32_t chunk_max = compute_batch(num_rows, FIFO_BATCH_FRAC); + uint32_t n_chunks = (num_rows + chunk_max - 1) / chunk_max; + + /* Workspace: [M4 | M3_pp[0] | M3_pp[1] | R2_pp[0] | R2_pp[1]] */ + uint32_t obi_m4 = ws; + uint32_t obi_m3[2] = {obi_m4 + DIM_D * DIM_E * 2, + obi_m4 + DIM_D * DIM_E * 2 + chunk_max * DIM_D * 2}; + uint32_t obi_r2[2] = {obi_m3[1] + chunk_max * DIM_D * 2, + obi_m3[1] + chunk_max * DIM_D * 2 + chunk_max * DIM_E * 2}; + + PERF_DELTA(cyc_preamble, _t0); + + /* Load full M4 from L2 */ + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m4_inp, obi_m4, DIM_D * DIM_E * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + + /* Prime: load chunk 0 of M3 */ + uint32_t c0s, c0l; + compute_chunk(num_rows, n_chunks, 0, &c0s, &c0l); + + uint32_t cyc_cmi_pre_prime = cyc_cmi; + + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m3_inp + (start_row + c0s) * DIM_D * 2, + obi_m3[0], + c0l * DIM_D * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + + uint32_t r2_batch = compute_batch(DIM_C, FIFO_BATCH_FRAC); + + for (uint32_t i = 0; i < n_chunks; i++) { + uint32_t cs, cl; + compute_chunk(num_rows, n_chunks, i, &cs, &cl); + uint32_t pp = i & 1u; + uint32_t has_next = (i + 1u < n_chunks); + + if (has_next) { + uint32_t ns, nl; + compute_chunk(num_rows, n_chunks, i + 1, &ns, &nl); + uint32_t cyc_cmi_before = cyc_cmi; + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m3_inp + (start_row + ns) * DIM_D * 2, + obi_m3[(i + 1) & 1u], + nl * DIM_D * 2); + PERF_DELTA(cyc_cmi, _t0); + } + + /* Compute R2 chunk i (zero-init + GEMM issue + wait) */ + uint32_t cyc_compute_before = cyc_compute; + _t0 = perf_get_cycles(); + mem_set_zero(obi_r2[pp], cl * DIM_E); + redmule_gemm(&redmule_ctrl, + obi_m3[pp], + obi_m4, + obi_r2[pp], + (uint16_t)cl, + (uint16_t)DIM_D, + (uint16_t)DIM_E); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_compute, _t0); + + if (has_next) { + uint32_t cyc_cmi_wait_before = cyc_cmi; + _t0 = perf_get_cycles(); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + } + + /* + * Push this chunk's R2 rows to every GEMM3 tile (all need + * the full R2). row_index = global row in the full R2 + * matrix (0..DIM_C-1). + */ + for (uint32_t r = 0; r < cl; r += r2_batch) { + uint32_t batch = (r + r2_batch <= cl) ? r2_batch : (cl - r); + uint32_t src_off = r * DIM_E * 2; + uint32_t global_r = start_row + cs + r; + for (uint32_t j = 0; j < GEMM3_N_TILES; j++) { + /* R2 slots start after R1 slots in the consumer's FIFO */ + uint32_t g3s_j, g3n_j; + get_row_range(j, GEMM3_N_TILES, DIM_A, &g3s_j, &g3n_j); + uint32_t slot_idx = g3n_j + global_r; + + uint32_t cyc_push_before = cyc_push; + fifo_push_dma(gemm3_tiles[j], + slot_idx, + obi_r2[pp] + src_off, + batch * DIM_E * 2, + MATRIX_R2, + global_r, + &idma_ctrl, + &eu_ctrl, + &cyc_push); + } + } + } + } + } + + /* ------------------------------------------------------------------ */ + /* GEMM3: incremental partial GEMM with K-dimension accumulation */ + /* ------------------------------------------------------------------ */ + if (gemm3_idx >= 0) { + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + + uint32_t start_row, num_rows; + get_row_range((uint32_t)gemm3_idx, GEMM3_N_TILES, DIM_A, &start_row, &num_rows); + + PERF_DELTA(cyc_preamble, _t0); + + _t0 = perf_get_cycles(); + + /* + * Workspace layout: + * [R2 (full DIM_C x DIM_E) | R3_slice (num_rows x DIM_E) | R1_tmp] + * + * R1 data is used directly from FIFO payload pointers. + * R1_tmp is a scratch buffer for column extraction when batch_rows > 1. + */ + uint32_t obi_r2 = ws; + uint32_t obi_r3 = obi_r2 + DIM_C * DIM_E * 2; + + /* Batch size for R1/R3 rows owned by this tile */ + uint32_t my_r1_batch = compute_batch(num_rows, FIFO_BATCH_FRAC); + uint32_t obi_r1_tmp = obi_r3 + num_rows * DIM_E * 2; + + /* + * Tracking state for incremental processing. + * r1_received[lr] : 1 once R1 batch at local row lr has arrived + * r1_data_ptrs[lr] : FIFO payload pointer for the R1 batch at lr + * r2_received[k] : 1 once R2 global row k has been DMA'd to workspace + * r3_k_done[lr] : number of K-columns accumulated into R3 for batch lr + * r3_pushed[lr] : 1 once R3 for batch lr has been pushed to GEMM4 + */ + uint8_t r1_received[DIM_A]; + uint32_t r1_data_ptrs[DIM_A]; + uint8_t r2_received[DIM_C]; + uint32_t r3_k_done[DIM_A]; + uint8_t r3_pushed[DIM_A]; + + for (uint32_t i = 0; i < DIM_A; i++) { + r1_received[i] = 0; + r1_data_ptrs[i] = 0; + r3_k_done[i] = 0; + r3_pushed[i] = 0; + } + for (uint32_t i = 0; i < DIM_C; i++) + r2_received[i] = 0; + + PERF_DELTA(cyc_preamble, _t0); + + uint32_t total_r3_done = 0; + + /* Zero R3 once; redmule_gemm accumulates (Y = X*W + Y) */ + uint32_t cyc_compute_zero_before = cyc_compute; + _t0 = perf_get_cycles(); + mem_set_zero(obi_r3, num_rows * DIM_E); + PERF_DELTA(cyc_compute, _t0); + + /* + * Spin until all R3 rows for this tile have been computed and pushed. + * GEMM3 tiles with num_rows == 0 skip this loop entirely. + */ + while (total_r3_done < num_rows) { + uint32_t data_ptr, data_size, matrix_id, row_index; + + uint32_t cyc_spin_before = cyc_spin; + + /* Spin until a FIFO message arrives */ + _t0 = perf_get_cycles(); + while (!fifo_pop(hartid, &data_ptr, &data_size, &matrix_id, &row_index)) + ; + PERF_DELTA(cyc_spin, _t0); + + if (matrix_id == MATRIX_R1) { + uint32_t cyc_compute_r1_before = cyc_compute; + uint32_t cyc_push_r1_before = cyc_push; + + /* + * Received a batch of R1 rows. + * Accumulate partial R3 for every contiguous group of R2 + * rows that have already arrived. + */ + uint32_t local_row = row_index - start_row; + uint32_t batch_rows = data_size / (DIM_C * 2); + r1_data_ptrs[local_row] = data_ptr; + r1_received[local_row] = 1; + + /* Scan r2_received[] for contiguous groups */ + uint32_t k = 0; + while (k < DIM_C) { + if (!r2_received[k]) { + k++; + continue; + } + uint32_t k_start = k; + while (k < DIM_C && r2_received[k]) + k++; + + total_r3_done += gemm3_partial_accum(local_row, + k_start, + k - k_start, + batch_rows, + data_ptr, + obi_r2, + obi_r3, + obi_r1_tmp, + start_row, + r3_k_done, + r3_pushed, + &redmule_ctrl, + &eu_ctrl, + &idma_ctrl, + &cyc_compute, + &cyc_push); + } + + } else if (matrix_id == MATRIX_R2) { + uint32_t cyc_cmi_r2_before = cyc_cmi; + uint32_t cyc_compute_r2_before = cyc_compute; + uint32_t cyc_push_r2_before = cyc_push; + + /* + * Received a batch of R2 rows. Copy to workspace and + * immediately accumulate against every R1 batch already present. + */ + uint32_t batch_rows = data_size / (DIM_E * 2); + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, 1, obi_r2 + row_index * DIM_E * 2, data_ptr, data_size); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + + for (uint32_t i = 0; i < batch_rows; i++) + r2_received[row_index + i] = 1; + + /* Accumulate this R2 batch against all received R1 batches */ + for (uint32_t lr = 0; lr < num_rows; lr += my_r1_batch) { + if (!r1_received[lr] || r3_pushed[lr]) + continue; + + uint32_t remaining = num_rows - lr; + uint32_t batch_r1 = (remaining < my_r1_batch) ? remaining : my_r1_batch; + + total_r3_done += gemm3_partial_accum(lr, + row_index, + batch_rows, + batch_r1, + r1_data_ptrs[lr], + obi_r2, + obi_r3, + obi_r1_tmp, + start_row, + r3_k_done, + r3_pushed, + &redmule_ctrl, + &eu_ctrl, + &idma_ctrl, + &cyc_compute, + &cyc_push); + } + } + } + } + + /* ------------------------------------------------------------------ */ + /* GEMM4: prefetch M5 into L1, then enter FIFO consumer */ + /* */ + /* Output is double-buffered: each batch writes to its own slice of */ + /* obi_o (indexed by local_row), so successive O DMAs never alias. */ + /* We issue the L1→L2 write non-blocking and overlap it with the */ + /* next batch's pop+compute, then wait before issuing the next write. */ + /* ------------------------------------------------------------------ */ + if (gemm4_idx >= 0) { + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + + uint32_t start_row, num_rows; + get_row_range((uint32_t)gemm4_idx, GEMM4_N_TILES, DIM_A, &start_row, &num_rows); + + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + + if (num_rows > 0) { + PERF_DELTA(cyc_preamble, _t0); + _t0 = perf_get_cycles(); + /* + * Workspace layout: [M5 | O_slice] + * M5 is placed first so Phase 3 only needs to know ws to find it. + */ + uint32_t obi_m5 = ws; + uint32_t obi_o = obi_m5 + DIM_E * DIM_F * 2; + + uint32_t cyc_cmi_m5_before = cyc_cmi; + /* Prefetch full M5 from L2 */ + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m5_inp, obi_m5, DIM_E * DIM_F * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + + /* + * FIFO consumer loop (Phase 3, data-driven, no explicit barrier). + * Spin until all expected R3 rows (== num_rows) have been processed. + */ + uint32_t total_o_done = 0; + uint32_t out_inflight = 0; /* 1 once an O write has been issued and not yet waited */ + + while (total_o_done < num_rows) { + uint32_t data_ptr, data_size, matrix_id, row_index; + + uint32_t cyc_spin_before = cyc_spin; + /* Spin until a message arrives */ + _t0 = perf_get_cycles(); + while (!fifo_pop(hartid, &data_ptr, &data_size, &matrix_id, &row_index)) + ; + PERF_DELTA(cyc_spin, _t0); + + uint32_t batch_rows = data_size / (DIM_E * 2); + uint32_t local_row = row_index - start_row; + uint32_t obi_o_batch = obi_o + local_row * DIM_F * 2; + + /* Compute O_batch = R3_batch @ M5; use payload directly as R3 input */ + uint32_t cyc_compute_before = cyc_compute; + _t0 = perf_get_cycles(); + mem_set_zero(obi_o_batch, batch_rows * DIM_F); + redmule_gemm(&redmule_ctrl, + data_ptr, + obi_m5, + obi_o_batch, + (uint16_t)batch_rows, + (uint16_t)DIM_E, + (uint16_t)DIM_F); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_compute, _t0); + + /* Wait for previous O write to complete before issuing the next */ + uint32_t cyc_cmo_wait_before = cyc_cmo; + if (out_inflight) { + _t0 = perf_get_cycles(); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmo, _t0); + out_inflight = 0; + } + + uint32_t cyc_cmo_issue_before = cyc_cmo; + /* Issue O batch write to L2 (non-blocking; overlaps with next compute) */ + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 1, + (uint32_t)o_out + row_index * DIM_F * 2, + obi_o_batch, + batch_rows * DIM_F * 2); + PERF_DELTA(cyc_cmo, _t0); + out_inflight = 1; + + total_o_done += batch_rows; + } + + uint32_t cyc_cmo_drain_before = cyc_cmo; + /* Drain the final outstanding O write before the closing barrier */ + if (out_inflight) { + _t0 = perf_get_cycles(); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmo, _t0); + } + } + } + + /* ------------------------------------------------------------------ */ + /* Final barrier: wait for all tiles to finish before validation */ + /* ------------------------------------------------------------------ */ + uint32_t cyc_barrier_final_before = cyc_barrier; + _t0 = perf_get_cycles(); + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_barrier, _t0); + + perf_stop(); + + if (hartid == 0) { + printf("====================== TOTAL CYCLES: %u ======================\n", + perf_get_cycles()); + } + + /* Validation: tile 0 checks O against golden */ + uint32_t errors = 0; + + if (hartid == 0) { + for (uint32_t i = 0; i < DIM_A; i++) { + for (uint32_t j = 0; j < DIM_F; j++) { + float16 computed = *(volatile float16 *)(&o_out[i * DIM_F + j]); + float16 expected = o_golden[i * DIM_F + j]; + + uint16_t uc = *(uint16_t *)&computed; + uint16_t ue = *(uint16_t *)&expected; + + int32_t vc = fp16_to_millis(uc); + int32_t ve = fp16_to_millis(ue); + + int32_t abs_diff = vc - ve; + if (abs_diff < 0) + abs_diff = -abs_diff; + if (abs_diff > abs_threshold_millis) { + errors++; + printf("O[%d][%d]: got=%f (0x%x) exp=%f (0x%x) (abs_diff=%ld)\n", + i, + j, + fp16_to_f64(uc), + uc, + fp16_to_f64(ue), + ue, + (long)abs_diff); + } + } + } + + printf("\nTest complete. Errors: %d / %d\n\n", errors, DIM_A * DIM_F); + } + + return errors; +} diff --git a/tests/magia/mesh/gemm_comm/fifo/src/test_with_prints.c b/tests/magia/mesh/gemm_comm/fifo/src/test_with_prints.c new file mode 100644 index 00000000..c6e95984 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/fifo/src/test_with_prints.c @@ -0,0 +1,1151 @@ +#include +#include "test.h" + +#include "tile.h" +#include "gemm_utils.h" +#include "fsync.h" +#include "idma.h" +#include "redmule.h" +#include "eventunit.h" +#include "utils/l1_fifo.h" + +#define WAIT_MODE WFE + +/** + * Tile group definitions for 8x8 mesh (64 tiles). + * + * GEMM1 tiles: [0, 1, 8, 9] (4 tiles, top-left 2x2 block) + * GEMM2 tiles: [16-19, 24-27, 32-35, + * 40-43, 48-51, 56-59] (24 tiles, rows 2-7 cols 0-3) + * GEMM3 tiles: [2-7, 10-15] (12 tiles, rows 0-1 cols 2-7) + * GEMM4 tiles: [20-23, 28-31, 36-39, + * 44-47, 52-55, 60-63] (24 tiles, rows 2-7 cols 4-7) + */ +#define GEMM1_N_TILES 4 +#define GEMM2_N_TILES 24 +#define GEMM3_N_TILES 12 +#define GEMM4_N_TILES 24 + +#define abs_threshold_millis 8 /* 0.008 expressed as integer millis */ + +/* FIFO message type tags */ +#define MATRIX_R1 0u +#define MATRIX_R2 1u +#define MATRIX_R3 2u + +/* + * 16 KB reserved at the start of each tile's L1 for the FIFO header, + * linked-list nodes, and payload data. Workspace buffers follow after. + */ +#define FIFO_RESERVE_SIZE 0x4000u + +/* + * Fraction of a tile's rows to push per fifo_push_dma() call. + * Set at build time via -DFIFO_BATCH_FRAC=F (default 0.2, i.e. 5 steps of 20%). + * If <= 0, falls back to one row at a time. + */ +#ifndef FIFO_BATCH_FRAC +#define FIFO_BATCH_FRAC 0.2f +#endif + +/* + * Compute the batch size (rows per push) for a tile with `total` rows. + * Returns ceil(total * frac), minimum 1. + * If frac <= 0, returns 1 (one row at a time). + */ +static uint32_t compute_batch(uint32_t total, float frac) +{ + if (frac <= 0.0f || total == 0) + return 1; + uint32_t b = (uint32_t)(total * frac + 0.9999f); /* ceiling */ + return (b < 1) ? 1 : b; +} + +/* + * Compute chunk i out of n_chunks for splitting `total` row count. + * Distributes the remainder across the first chunks (mirrors get_row_range). + */ +static void +compute_chunk(uint32_t total, uint32_t n_chunks, uint32_t idx, uint32_t *start, uint32_t *len) +{ + uint32_t base = total / n_chunks; + uint32_t rem = total % n_chunks; + *start = idx * base + (idx < rem ? idx : rem); + *len = base + (idx < rem ? 1 : 0); +} + +static const uint32_t gemm1_tiles[GEMM1_N_TILES] = {0, 1, 8, 9}; + +static const uint32_t gemm2_tiles[GEMM2_N_TILES] = {16, 17, 18, 19, 24, 25, 26, 27, 32, 33, 34, 35, + 40, 41, 42, 43, 48, 49, 50, 51, 56, 57, 58, 59}; + +static const uint32_t gemm3_tiles[GEMM3_N_TILES] = {2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15}; + +static const uint32_t gemm4_tiles[GEMM4_N_TILES] = {20, 21, 22, 23, 28, 29, 30, 31, 36, 37, 38, 39, + 44, 45, 46, 47, 52, 53, 54, 55, 60, 61, 62, 63}; + +static int get_local_idx(uint32_t hartid, const uint32_t *tiles, uint32_t n_tiles) +{ + for (uint32_t i = 0; i < n_tiles; i++) + if (tiles[i] == hartid) + return (int)i; + return -1; +} + +static void get_row_range(uint32_t local_idx, + uint32_t n_tiles, + uint32_t total_rows, + uint32_t *start_row, + uint32_t *num_rows) +{ + uint32_t base = total_rows / n_tiles; + uint32_t rem = total_rows % n_tiles; + *start_row = local_idx * base + (local_idx < rem ? local_idx : rem); + *num_rows = base + (local_idx < rem ? 1 : 0); +} + +static void mem_set_zero(uint32_t o, uint32_t dim) +{ + for (uint32_t i = 0; i < dim; i++) + mmio16(o + i * 2) = 0x0000; +} + +/* + * Push data to a target tile's FIFO slot using iDMA for the payload transfer. + */ +static void fifo_push_dma(uint32_t target_hartid, + uint32_t slot_idx, + uint32_t src_addr, + uint32_t size_bytes, + uint32_t matrix_id, + uint32_t row_index, + idma_controller_t *idma_ctrl, + eu_controller_t *eu_ctrl, + uint32_t *cyc_push) +{ + fifo_header_t *hdr = fifo_get_header(target_hartid); + fifo_slot_t *slot = fifo_get_slot(hdr, slot_idx); + uint32_t dst = (uint32_t)fifo_slot_data(slot); + + uint32_t t0 = perf_get_cycles(); + /* DMA payload: local L1 (OBI) → remote L1 (AXI) */ + idma_memcpy_1d(idma_ctrl, 1, dst, src_addr, size_bytes); + eu_idma_wait_o2a(eu_ctrl, WAIT_MODE); + + /* Publish metadata + valid flag */ + fifo_slot_publish(target_hartid, slot_idx, size_bytes, matrix_id, row_index); + PERF_DELTA(*cyc_push, t0); +} + +/* + * Push completed R3 rows to all overlapping GEMM4 tile FIFOs. + */ +static void push_r3_to_gemm4(uint32_t global_row, + uint32_t batch_rows, + uint32_t obi_r3_batch, + idma_controller_t *idma_ctrl, + eu_controller_t *eu_ctrl, + uint32_t *cyc_push) +{ + for (uint32_t k = 0; k < GEMM4_N_TILES; k++) { + uint32_t g4_start, g4_nrows; + get_row_range(k, GEMM4_N_TILES, DIM_A, &g4_start, &g4_nrows); + if (g4_nrows == 0) + continue; + + uint32_t ov_start = global_row > g4_start ? global_row : g4_start; + uint32_t ov_end = (global_row + batch_rows) < (g4_start + g4_nrows) + ? (global_row + batch_rows) + : (g4_start + g4_nrows); + if (ov_start >= ov_end) + continue; + + uint32_t ov_count = ov_end - ov_start; + uint32_t src_off = (ov_start - global_row) * DIM_E * 2; + uint32_t slot_idx = ov_start - g4_start; + fifo_push_dma(gemm4_tiles[k], + slot_idx, + obi_r3_batch + src_off, + ov_count * DIM_E * 2, + MATRIX_R3, + ov_start, + idma_ctrl, + eu_ctrl, + cyc_push); + } +} + +/* + * Accumulate a partial R3 contribution for R1 batch at local row lr, + * using R2 rows [k_start .. k_start + k_len). + * + * Returns the number of newly completed R3 rows (pushed to GEMM4), or 0. + */ +static uint32_t gemm3_partial_accum(uint32_t lr, + uint32_t k_start, + uint32_t k_len, + uint32_t r1_batch, + uint32_t r1_ptr, + uint32_t obi_r2, + uint32_t obi_r3, + uint32_t obi_r1_tmp, + uint32_t start_row, + uint32_t *r3_k_done, + uint8_t *r3_pushed, + redmule_controller_t *redmule_ctrl, + eu_controller_t *eu_ctrl, + idma_controller_t *idma_ctrl, + uint32_t *cyc_compute, + uint32_t *cyc_push) +{ + uint32_t A_ptr; + + if (r1_batch == 1) { + /* Single row: R1[0, k_start:k_start+k_len] is contiguous within the row */ + A_ptr = r1_ptr + k_start * 2; + } else { + /* Multiple rows: extract columns k_start..k_start+k_len into temp buffer */ + for (uint32_t r = 0; r < r1_batch; r++) { + volatile uint16_t *src = (volatile uint16_t *)(r1_ptr + (r * DIM_C + k_start) * 2); + volatile uint16_t *dst = (volatile uint16_t *)(obi_r1_tmp + r * k_len * 2); + for (uint32_t c = 0; c < k_len; c++) + dst[c] = src[c]; + } + A_ptr = obi_r1_tmp; + } + + uint32_t B_ptr = obi_r2 + k_start * DIM_E * 2; + uint32_t obi_r3_batch = obi_r3 + lr * DIM_E * 2; + + uint32_t t0 = perf_get_cycles(); + redmule_gemm(redmule_ctrl, + A_ptr, + B_ptr, + obi_r3_batch, + (uint16_t)r1_batch, + (uint16_t)k_len, + (uint16_t)DIM_E); + eu_redmule_wait(eu_ctrl, WAIT_MODE); + PERF_DELTA(*cyc_compute, t0); + + r3_k_done[lr] += k_len; + + if (r3_k_done[lr] == DIM_C && !r3_pushed[lr]) { + r3_pushed[lr] = 1; + uint32_t global_row = start_row + lr; + push_r3_to_gemm4(global_row, r1_batch, obi_r3_batch, idma_ctrl, eu_ctrl, cyc_push); + return r1_batch; + } + + return 0; +} + +int main(void) +{ + /* 0. Initialization */ + uint32_t hartid = get_hartid(); + uint32_t l1_tile_base = get_l1_base(hartid); + + /* Init iDMA */ + idma_config_t idma_cfg = {.hartid = hartid}; + idma_controller_t idma_ctrl = { + .base = NULL, + .cfg = &idma_cfg, + .api = &idma_api, + }; + idma_init(&idma_ctrl); + + /* Init RedMulE */ + redmule_config_t redmule_cfg = {.hartid = hartid}; + redmule_controller_t redmule_ctrl = { + .base = NULL, + .cfg = &redmule_cfg, + .api = &redmule_api, + }; + redmule_init(&redmule_ctrl); + + /* Init FractalSync */ + fsync_config_t fsync_cfg = {.hartid = hartid}; + fsync_controller_t fsync_ctrl = { + .base = NULL, + .cfg = &fsync_cfg, + .api = &fsync_api, + }; + fsync_init(&fsync_ctrl); + +/* Init Event Unit */ +#if STALLING == 0 + eu_config_t eu_cfg = {.hartid = hartid}; + eu_controller_t eu_ctrl = { + .base = NULL, + .cfg = &eu_cfg, + .api = &eu_api, + }; + + eu_init(&eu_ctrl); + eu_clear_events(0xFFFFFFFF); + eu_fsync_init(&eu_ctrl, 0); + eu_idma_init(&eu_ctrl, 0); + eu_redmule_init(&eu_ctrl, 0); +#endif + + /* Per-tile cycle-count breakdown (accumulated across phases) */ + uint32_t cyc_barrier = 0; /* fsync barrier wait */ + uint32_t cyc_preamble = 0; /* GEMM1 preamble (setup before first load), excluding prints */ + uint32_t cyc_cmi = 0; /* input DMA: L2→L1 matrix loads + GEMM3 R2 workspace copy */ + uint32_t cyc_compute = 0; /* RedMulE GEMM time */ + uint32_t cyc_push = 0; /* fifo_push_dma: DMA + publish */ + uint32_t cyc_spin = 0; /* fifo_pop spin (consumer idle) */ + uint32_t cyc_cmo = 0; /* output DMA: L1→L2 (GEMM4 only) */ + uint32_t _t0; + + /* Determine group membership for this tile (needed to size the FIFO) */ + int gemm1_idx = get_local_idx(hartid, gemm1_tiles, GEMM1_N_TILES); + int gemm2_idx = get_local_idx(hartid, gemm2_tiles, GEMM2_N_TILES); + int gemm3_idx = get_local_idx(hartid, gemm3_tiles, GEMM3_N_TILES); + int gemm4_idx = get_local_idx(hartid, gemm4_tiles, GEMM4_N_TILES); + + /* + * Initialize this tile's FIFO before the startup barrier. + * Slot count and payload size depend on the tile's consumer role. + * Only needed for GEMM 3 and GEMM 4 (which have as input the result + * of other GEMMs, performed by other tiles). + */ + uint32_t num_slots = 0; + uint32_t slot_data_size = 0; + + if (gemm3_idx >= 0) { + uint32_t g3s, g3n; + + get_row_range((uint32_t)gemm3_idx, GEMM3_N_TILES, DIM_A, &g3s, &g3n); + + uint32_t r1_batch = compute_batch(g3n, FIFO_BATCH_FRAC); + uint32_t r2_batch = compute_batch(DIM_C, FIFO_BATCH_FRAC); + + uint32_t r1_payload = r1_batch * DIM_C * 2; + uint32_t r2_payload = r2_batch * DIM_E * 2; + + num_slots = g3n + DIM_C; + slot_data_size = r1_payload > r2_payload ? r1_payload : r2_payload; + } else if (gemm4_idx >= 0) { + uint32_t g4s, g4n; + + get_row_range((uint32_t)gemm4_idx, GEMM4_N_TILES, DIM_A, &g4s, &g4n); + + uint32_t r3_batch = compute_batch(g4n, FIFO_BATCH_FRAC); + + num_slots = g4n; + slot_data_size = r3_batch * DIM_E * 2; + } + + fifo_init(hartid, num_slots, slot_data_size); + + /* + * Startup barrier: ensure all tiles finish crt0.S BSS zeroing (which + * touches global output buffers) and FIFO initialization before any + * tile begins pushing into another tile's FIFO. + */ + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + perf_reset(); + perf_start(); + + /* + * L1 workspace starts at FIFO_RESERVE_SIZE bytes past the tile base. + * The first FIFO_RESERVE_SIZE bytes are reserved for the FIFO header + * and slot array written by remote pushers. + */ + uint32_t ws = l1_tile_base + FIFO_RESERVE_SIZE; + + printf("Tile %u: Initialization completed.", hartid); + _t0 = perf_get_cycles(); + + /* ------------------------------------------------------------------ */ + /* GEMM1: R1 = M1 @ M2, push R1 rows to GEMM3 FIFOs */ + /* */ + /* Double-buffered: split the local M1 row slice into chunks of */ + /* compute_batch(num_rows, FIFO_BATCH_FRAC) rows. While RedMulE */ + /* computes chunk i, iDMA prefetches chunk i+1 into the alternate */ + /* ping-pong buffer. M2 is loaded once and shared across all chunks. */ + /* ------------------------------------------------------------------ */ + if (gemm1_idx >= 0) { + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: Starting GEMM1...\n", hartid); + _t0 = perf_get_cycles(); + + uint32_t start_row, num_rows; + get_row_range((uint32_t)gemm1_idx, GEMM1_N_TILES, DIM_A, &start_row, &num_rows); + + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: GEMM1 computed rows [%u, %u)\n", hartid, start_row, start_row + num_rows); + _t0 = perf_get_cycles(); + + if (num_rows > 0) { + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: GEMM1 splitting into batches\n", hartid); + _t0 = perf_get_cycles(); + + uint32_t chunk_max = compute_batch(num_rows, FIFO_BATCH_FRAC); + uint32_t n_chunks = (num_rows + chunk_max - 1) / chunk_max; + + /* Workspace: [M2 | M1_pp[0] | M1_pp[1] | R1_pp[0] | R1_pp[1]] */ + uint32_t obi_m2 = ws; + uint32_t obi_m1[2] = {obi_m2 + DIM_B * DIM_C * 2, + obi_m2 + DIM_B * DIM_C * 2 + chunk_max * DIM_B * 2}; + uint32_t obi_r1[2] = {obi_m1[1] + chunk_max * DIM_B * 2, + obi_m1[1] + chunk_max * DIM_B * 2 + chunk_max * DIM_C * 2}; + + PERF_DELTA(cyc_preamble, _t0); + printf( + "Tile %u GEMM1: Workspace prepared. Working on %u batches (preamble: %u cycles)\n ", + hartid, + n_chunks, + cyc_preamble); + + /* Load full M2 from L2 */ + printf("Tile %u GEMM1: Loading M2[%d x %d]...\n", hartid, DIM_B, DIM_C); + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m2_inp, obi_m2, DIM_B * DIM_C * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + printf( + "Tile %u GEMM1: M2[%d x %d] loaded (cycles: %u)\n", hartid, DIM_B, DIM_C, cyc_cmi); + + /* Prime the pipeline: load M1_pp[0] */ + printf("Tile %u GEMM1: Priming pipeline with M1 chunk 0...\n", hartid); + uint32_t c0s, c0l; + compute_chunk(num_rows, n_chunks, 0, &c0s, &c0l); + + uint32_t cyc_cmi_pre_prime = cyc_cmi; + printf("Tile %u GEMM1: priming load preamble completed (cycles: %u)\n", + hartid, + cyc_cmi_pre_prime); + + printf("Tile %u GEMM1: Loading M1 rows [%u, %u)...\n", + hartid, + start_row + c0s, + start_row + c0s + c0l); + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m1_inp + (start_row + c0s) * DIM_B * 2, + obi_m1[0], + c0l * DIM_B * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + printf("Tile %u GEMM1: M1 chunk 0 loaded (cycles: %u)\n", hartid, cyc_cmi); + + for (uint32_t i = 0; i < n_chunks; i++) { + printf("Tile %u GEMM1: Starting batch %u/%u...\n", hartid, i + 1, n_chunks); + + uint32_t cs, cl; + compute_chunk(num_rows, n_chunks, i, &cs, &cl); + uint32_t pp = i & 1u; + uint32_t has_next = (i + 1u < n_chunks); + + /* Kick off chunk i+1 load (overlaps with chunk i compute) */ + if (has_next) { + uint32_t ns, nl; + compute_chunk(num_rows, n_chunks, i + 1, &ns, &nl); + uint32_t cyc_cmi_before = cyc_cmi; + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m1_inp + (start_row + ns) * DIM_B * 2, + obi_m1[(i + 1) & 1u], + nl * DIM_B * 2); + PERF_DELTA(cyc_cmi, _t0); + + printf("Tile %u GEMM1: chunk %u/%u prefetch issue cycles: %u\n", + hartid, + i + 1, + n_chunks, + cyc_cmi - cyc_cmi_before); + } + + /* Compute R1 chunk i (zero-init + GEMM issue + wait) */ + uint32_t cyc_compute_before = cyc_compute; + _t0 = perf_get_cycles(); + mem_set_zero(obi_r1[pp], cl * DIM_C); + redmule_gemm(&redmule_ctrl, + obi_m1[pp], + obi_m2, + obi_r1[pp], + (uint16_t)cl, + (uint16_t)DIM_B, + (uint16_t)DIM_C); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_compute, _t0); + + printf("Tile %u GEMM1: chunk %u/%u compute cycles: %u\n", + hartid, + i + 1, + n_chunks, + cyc_compute - cyc_compute_before); + + if (has_next) { + uint32_t cyc_cmi_wait_before = cyc_cmi; + _t0 = perf_get_cycles(); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + + printf("Tile %u GEMM1: chunk %u/%u prefetch wait cycles: %u\n", + hartid, + i + 1, + n_chunks, + cyc_cmi - cyc_cmi_wait_before); + } + + /* + * Push this chunk's R1 rows to each GEMM3 tile that owns + * overlapping rows. Producer-side window for the chunk is + * [chunk_g_start, chunk_g_end). + */ + uint32_t chunk_g_start = start_row + cs; + uint32_t chunk_g_end = chunk_g_start + cl; + for (uint32_t j = 0; j < GEMM3_N_TILES; j++) { + uint32_t g3_start, g3_nrows; + get_row_range(j, GEMM3_N_TILES, DIM_A, &g3_start, &g3_nrows); + if (g3_nrows == 0) + continue; + + uint32_t ov_start = chunk_g_start > g3_start ? chunk_g_start : g3_start; + uint32_t ov_end = + chunk_g_end < (g3_start + g3_nrows) ? chunk_g_end : (g3_start + g3_nrows); + if (ov_start >= ov_end) + continue; + + uint32_t g3_r1_batch = compute_batch(g3_nrows, FIFO_BATCH_FRAC); + for (uint32_t r = ov_start; r < ov_end; r += g3_r1_batch) { + uint32_t batch = (r + g3_r1_batch <= ov_end) ? g3_r1_batch : (ov_end - r); + uint32_t src_off = (r - chunk_g_start) * DIM_C * 2; + uint32_t slot_idx = r - g3_start; + + uint32_t cyc_push_before = cyc_push; + fifo_push_dma(gemm3_tiles[j], + slot_idx, + obi_r1[pp] + src_off, + batch * DIM_C * 2, + MATRIX_R1, + r, + &idma_ctrl, + &eu_ctrl, + &cyc_push); + + printf("Tile %u GEMM1: chunk %u/%u push to tile %u row %u cycles: %u\n", + hartid, + i + 1, + n_chunks, + gemm3_tiles[j], + r, + cyc_push - cyc_push_before); + } + } + } + } + } + + /* ------------------------------------------------------------------ */ + /* GEMM2: R2 = M3 @ M4, push R2 rows to all GEMM3 FIFOs */ + /* */ + /* Double-buffered using the same chunk strategy as GEMM1. M4 is */ + /* loaded once; M3 chunks are ping-pong'd against RedMulE compute. */ + /* ------------------------------------------------------------------ */ + if (gemm2_idx >= 0) { + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: Starting GEMM2...\n", hartid); + _t0 = perf_get_cycles(); + + uint32_t start_row, num_rows; + get_row_range((uint32_t)gemm2_idx, GEMM2_N_TILES, DIM_C, &start_row, &num_rows); + + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: GEMM2 computed rows [%u, %u)\n", hartid, start_row, start_row + num_rows); + _t0 = perf_get_cycles(); + + if (num_rows > 0) { + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: GEMM2 splitting into batches\n", hartid); + _t0 = perf_get_cycles(); + + uint32_t chunk_max = compute_batch(num_rows, FIFO_BATCH_FRAC); + uint32_t n_chunks = (num_rows + chunk_max - 1) / chunk_max; + + /* Workspace: [M4 | M3_pp[0] | M3_pp[1] | R2_pp[0] | R2_pp[1]] */ + uint32_t obi_m4 = ws; + uint32_t obi_m3[2] = {obi_m4 + DIM_D * DIM_E * 2, + obi_m4 + DIM_D * DIM_E * 2 + chunk_max * DIM_D * 2}; + uint32_t obi_r2[2] = {obi_m3[1] + chunk_max * DIM_D * 2, + obi_m3[1] + chunk_max * DIM_D * 2 + chunk_max * DIM_E * 2}; + + PERF_DELTA(cyc_preamble, _t0); + printf( + "Tile %u GEMM2: Workspace prepared. Working on %u batches (preamble: %u cycles)\n ", + hartid, + n_chunks, + cyc_preamble); + + /* Load full M4 from L2 */ + printf("Tile %u GEMM2: Loading M4[%d x %d]...\n", hartid, DIM_D, DIM_E); + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m4_inp, obi_m4, DIM_D * DIM_E * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + printf( + "Tile %u GEMM2: M4[%d x %d] loaded (cycles: %u)\n", hartid, DIM_D, DIM_E, cyc_cmi); + + /* Prime: load chunk 0 of M3 */ + printf("Tile %u GEMM2: Priming pipeline with M3 chunk 0...\n", hartid); + uint32_t c0s, c0l; + compute_chunk(num_rows, n_chunks, 0, &c0s, &c0l); + + uint32_t cyc_cmi_pre_prime = cyc_cmi; + printf("Tile %u GEMM2: priming load preamble completed (cycles: %u)\n", + hartid, + cyc_cmi_pre_prime); + + printf("Tile %u GEMM2: Loading M3 rows [%u, %u)...\n", + hartid, + start_row + c0s, + start_row + c0s + c0l); + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m3_inp + (start_row + c0s) * DIM_D * 2, + obi_m3[0], + c0l * DIM_D * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + printf("Tile %u GEMM2: M3 chunk 0 loaded (cycles: %u)\n", hartid, cyc_cmi); + + uint32_t r2_batch = compute_batch(DIM_C, FIFO_BATCH_FRAC); + + for (uint32_t i = 0; i < n_chunks; i++) { + printf("Tile %u GEMM2: Starting batch %u/%u...\n", hartid, i + 1, n_chunks); + + uint32_t cs, cl; + compute_chunk(num_rows, n_chunks, i, &cs, &cl); + uint32_t pp = i & 1u; + uint32_t has_next = (i + 1u < n_chunks); + + if (has_next) { + uint32_t ns, nl; + compute_chunk(num_rows, n_chunks, i + 1, &ns, &nl); + uint32_t cyc_cmi_before = cyc_cmi; + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m3_inp + (start_row + ns) * DIM_D * 2, + obi_m3[(i + 1) & 1u], + nl * DIM_D * 2); + PERF_DELTA(cyc_cmi, _t0); + + printf("Tile %u GEMM2: chunk %u/%u prefetch issue cycles: %u\n", + hartid, + i + 1, + n_chunks, + cyc_cmi - cyc_cmi_before); + } + + /* Compute R2 chunk i (zero-init + GEMM issue + wait) */ + uint32_t cyc_compute_before = cyc_compute; + _t0 = perf_get_cycles(); + mem_set_zero(obi_r2[pp], cl * DIM_E); + redmule_gemm(&redmule_ctrl, + obi_m3[pp], + obi_m4, + obi_r2[pp], + (uint16_t)cl, + (uint16_t)DIM_D, + (uint16_t)DIM_E); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_compute, _t0); + + printf("Tile %u GEMM2: chunk %u/%u compute cycles: %u\n", + hartid, + i + 1, + n_chunks, + cyc_compute - cyc_compute_before); + + if (has_next) { + uint32_t cyc_cmi_wait_before = cyc_cmi; + _t0 = perf_get_cycles(); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + + printf("Tile %u GEMM2: chunk %u/%u prefetch wait cycles: %u\n", + hartid, + i + 1, + n_chunks, + cyc_cmi - cyc_cmi_wait_before); + } + + /* + * Push this chunk's R2 rows to every GEMM3 tile (all need + * the full R2). row_index = global row in the full R2 + * matrix (0..DIM_C-1). + */ + for (uint32_t r = 0; r < cl; r += r2_batch) { + uint32_t batch = (r + r2_batch <= cl) ? r2_batch : (cl - r); + uint32_t src_off = r * DIM_E * 2; + uint32_t global_r = start_row + cs + r; + for (uint32_t j = 0; j < GEMM3_N_TILES; j++) { + /* R2 slots start after R1 slots in the consumer's FIFO */ + uint32_t g3s_j, g3n_j; + get_row_range(j, GEMM3_N_TILES, DIM_A, &g3s_j, &g3n_j); + uint32_t slot_idx = g3n_j + global_r; + + uint32_t cyc_push_before = cyc_push; + fifo_push_dma(gemm3_tiles[j], + slot_idx, + obi_r2[pp] + src_off, + batch * DIM_E * 2, + MATRIX_R2, + global_r, + &idma_ctrl, + &eu_ctrl, + &cyc_push); + + printf("Tile %u GEMM2: chunk %u/%u push to tile %u row %u cycles: %u\n", + hartid, + i + 1, + n_chunks, + gemm3_tiles[j], + global_r, + cyc_push - cyc_push_before); + } + } + } + } + } + + /* ------------------------------------------------------------------ */ + /* GEMM3: incremental partial GEMM with K-dimension accumulation */ + /* ------------------------------------------------------------------ */ + if (gemm3_idx >= 0) { + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: Starting GEMM3...\n", hartid); + _t0 = perf_get_cycles(); + + uint32_t start_row, num_rows; + get_row_range((uint32_t)gemm3_idx, GEMM3_N_TILES, DIM_A, &start_row, &num_rows); + + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: GEMM3 computed rows [%u, %u)\n", hartid, start_row, start_row + num_rows); + _t0 = perf_get_cycles(); + + /* + * Workspace layout: + * [R2 (full DIM_C x DIM_E) | R3_slice (num_rows x DIM_E) | R1_tmp] + * + * R1 data is used directly from FIFO payload pointers. + * R1_tmp is a scratch buffer for column extraction when batch_rows > 1. + */ + uint32_t obi_r2 = ws; + uint32_t obi_r3 = obi_r2 + DIM_C * DIM_E * 2; + + /* Batch size for R1/R3 rows owned by this tile */ + uint32_t my_r1_batch = compute_batch(num_rows, FIFO_BATCH_FRAC); + uint32_t obi_r1_tmp = obi_r3 + num_rows * DIM_E * 2; + + /* + * Tracking state for incremental processing. + * r1_received[lr] : 1 once R1 batch at local row lr has arrived + * r1_data_ptrs[lr] : FIFO payload pointer for the R1 batch at lr + * r2_received[k] : 1 once R2 global row k has been DMA'd to workspace + * r3_k_done[lr] : number of K-columns accumulated into R3 for batch lr + * r3_pushed[lr] : 1 once R3 for batch lr has been pushed to GEMM4 + */ + uint8_t r1_received[DIM_A]; + uint32_t r1_data_ptrs[DIM_A]; + uint8_t r2_received[DIM_C]; + uint32_t r3_k_done[DIM_A]; + uint8_t r3_pushed[DIM_A]; + + for (uint32_t i = 0; i < DIM_A; i++) { + r1_received[i] = 0; + r1_data_ptrs[i] = 0; + r3_k_done[i] = 0; + r3_pushed[i] = 0; + } + for (uint32_t i = 0; i < DIM_C; i++) + r2_received[i] = 0; + + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u GEMM3: Workspace prepared. Expecting %u R3 rows (preamble: %u cycles)\n", + hartid, + num_rows, + cyc_preamble); + + uint32_t total_r3_done = 0; + + /* Zero R3 once; redmule_gemm accumulates (Y = X*W + Y) */ + uint32_t cyc_compute_zero_before = cyc_compute; + _t0 = perf_get_cycles(); + mem_set_zero(obi_r3, num_rows * DIM_E); + PERF_DELTA(cyc_compute, _t0); + printf("Tile %u GEMM3: R3 zero-init compute cycles: %u\n", + hartid, + cyc_compute - cyc_compute_zero_before); + + /* + * Spin until all R3 rows for this tile have been computed and pushed. + * GEMM3 tiles with num_rows == 0 skip this loop entirely. + */ + while (total_r3_done < num_rows) { + uint32_t data_ptr, data_size, matrix_id, row_index; + + uint32_t cyc_spin_before = cyc_spin; + + /* Spin until a FIFO message arrives */ + _t0 = perf_get_cycles(); + while (!fifo_pop(hartid, &data_ptr, &data_size, &matrix_id, &row_index)) + ; + PERF_DELTA(cyc_spin, _t0); + + printf("Tile %u GEMM3: spin cycles: %u (matrix_id=%u row_index=%u)\n", + hartid, + cyc_spin - cyc_spin_before, + matrix_id, + row_index); + + if (matrix_id == MATRIX_R1) { + uint32_t cyc_compute_r1_before = cyc_compute; + uint32_t cyc_push_r1_before = cyc_push; + + /* + * Received a batch of R1 rows. + * Accumulate partial R3 for every contiguous group of R2 + * rows that have already arrived. + */ + uint32_t local_row = row_index - start_row; + uint32_t batch_rows = data_size / (DIM_C * 2); + r1_data_ptrs[local_row] = data_ptr; + r1_received[local_row] = 1; + + printf("Tile %u GEMM3: Received R1 batch for local row %u (global row %u, batch " + "rows %u)\n", + hartid, + local_row, + row_index, + batch_rows); + + /* Scan r2_received[] for contiguous groups */ + uint32_t k = 0; + while (k < DIM_C) { + if (!r2_received[k]) { + k++; + continue; + } + uint32_t k_start = k; + while (k < DIM_C && r2_received[k]) + k++; + + printf("Tile %u GEMM3: Accumulating R1 batch into R3 for local row %u (global" + "row %u, " + "R2 global rows [%u, %u))\n", + hartid, + local_row, + row_index, + k_start, + k); + + total_r3_done += gemm3_partial_accum(local_row, + k_start, + k - k_start, + batch_rows, + data_ptr, + obi_r2, + obi_r3, + obi_r1_tmp, + start_row, + r3_k_done, + r3_pushed, + &redmule_ctrl, + &eu_ctrl, + &idma_ctrl, + &cyc_compute, + &cyc_push); + } + + printf("Tile %u GEMM3: R1 path compute cycles: %u, push cycles: %u\n", + hartid, + cyc_compute - cyc_compute_r1_before, + cyc_push - cyc_push_r1_before); + + } else if (matrix_id == MATRIX_R2) { + uint32_t cyc_cmi_r2_before = cyc_cmi; + uint32_t cyc_compute_r2_before = cyc_compute; + uint32_t cyc_push_r2_before = cyc_push; + + /* + * Received a batch of R2 rows. Copy to workspace and + * immediately accumulate against every R1 batch already present. + */ + uint32_t batch_rows = data_size / (DIM_E * 2); + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, 1, obi_r2 + row_index * DIM_E * 2, data_ptr, data_size); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + + for (uint32_t i = 0; i < batch_rows; i++) + r2_received[row_index + i] = 1; + + printf("Tile %u GEMM3: Received R2 batch for global row %u (batch rows %u, cycles " + "to DMA %u)\n ", + hartid, + row_index, + batch_rows, + cyc_cmi); + + /* Accumulate this R2 batch against all received R1 batches */ + for (uint32_t lr = 0; lr < num_rows; lr += my_r1_batch) { + if (!r1_received[lr] || r3_pushed[lr]) + continue; + + uint32_t remaining = num_rows - lr; + uint32_t batch_r1 = (remaining < my_r1_batch) ? remaining : my_r1_batch; + + printf( + "Tile %u GEMM3: Accumulating R2 batch into R3 for local row %u (global row " + "%u, R1 batch rows %u)\n", + hartid, + lr, + row_index, + batch_r1); + + total_r3_done += gemm3_partial_accum(lr, + row_index, + batch_rows, + batch_r1, + r1_data_ptrs[lr], + obi_r2, + obi_r3, + obi_r1_tmp, + start_row, + r3_k_done, + r3_pushed, + &redmule_ctrl, + &eu_ctrl, + &idma_ctrl, + &cyc_compute, + &cyc_push); + } + + printf("Tile %u GEMM3: R2 path cmi cycles: %u, compute cycles: %u, push cycles: % " + "u\n ", + hartid, + cyc_cmi - cyc_cmi_r2_before, + cyc_compute - cyc_compute_r2_before, + cyc_push - cyc_push_r2_before); + } + } + } + + /* ------------------------------------------------------------------ */ + /* GEMM4: prefetch M5 into L1, then enter FIFO consumer */ + /* */ + /* Output is double-buffered: each batch writes to its own slice of */ + /* obi_o (indexed by local_row), so successive O DMAs never alias. */ + /* We issue the L1→L2 write non-blocking and overlap it with the */ + /* next batch's pop+compute, then wait before issuing the next write. */ + /* ------------------------------------------------------------------ */ + if (gemm4_idx >= 0) { + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: Starting GEMM4...\n", hartid); + _t0 = perf_get_cycles(); + + uint32_t start_row, num_rows; + get_row_range((uint32_t)gemm4_idx, GEMM4_N_TILES, DIM_A, &start_row, &num_rows); + + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u: GEMM4 computed rows [%u, %u)\n", hartid, start_row, start_row + num_rows); + _t0 = perf_get_cycles(); + + if (num_rows > 0) { + PERF_DELTA(cyc_preamble, _t0); + printf("Tile %u GEMM4: Workspace prepared. Expecting %u O rows (preamble: %u cycles)\n", + hartid, + num_rows, + cyc_preamble); + _t0 = perf_get_cycles(); + /* + * Workspace layout: [M5 | O_slice] + * M5 is placed first so Phase 3 only needs to know ws to find it. + */ + uint32_t obi_m5 = ws; + uint32_t obi_o = obi_m5 + DIM_E * DIM_F * 2; + + uint32_t cyc_cmi_m5_before = cyc_cmi; + /* Prefetch full M5 from L2 */ + printf("Tile %u GEMM4: Loading M5[%d x %d]...\n", hartid, DIM_E, DIM_F); + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m5_inp, obi_m5, DIM_E * DIM_F * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmi, _t0); + printf("Tile %u GEMM4: M5[%d x %d] loaded (cycles: %u)\n", + hartid, + DIM_E, + DIM_F, + cyc_cmi - cyc_cmi_m5_before); + + /* + * FIFO consumer loop (Phase 3, data-driven, no explicit barrier). + * Spin until all expected R3 rows (== num_rows) have been processed. + */ + uint32_t total_o_done = 0; + uint32_t out_inflight = 0; /* 1 once an O write has been issued and not yet waited */ + + while (total_o_done < num_rows) { + uint32_t data_ptr, data_size, matrix_id, row_index; + + uint32_t cyc_spin_before = cyc_spin; + /* Spin until a message arrives */ + _t0 = perf_get_cycles(); + while (!fifo_pop(hartid, &data_ptr, &data_size, &matrix_id, &row_index)) + ; + PERF_DELTA(cyc_spin, _t0); + printf("Tile %u GEMM4: spin cycles: %u (matrix_id=%u row_index=%u)\n", + hartid, + cyc_spin - cyc_spin_before, + matrix_id, + row_index); + + uint32_t batch_rows = data_size / (DIM_E * 2); + uint32_t local_row = row_index - start_row; + uint32_t obi_o_batch = obi_o + local_row * DIM_F * 2; + + /* Compute O_batch = R3_batch @ M5; use payload directly as R3 input */ + uint32_t cyc_compute_before = cyc_compute; + _t0 = perf_get_cycles(); + mem_set_zero(obi_o_batch, batch_rows * DIM_F); + redmule_gemm(&redmule_ctrl, + data_ptr, + obi_m5, + obi_o_batch, + (uint16_t)batch_rows, + (uint16_t)DIM_E, + (uint16_t)DIM_F); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_compute, _t0); + printf("Tile %u GEMM4: O batch (row %u, %u rows) compute cycles: %u\n", + hartid, + row_index, + batch_rows, + cyc_compute - cyc_compute_before); + + /* Wait for previous O write to complete before issuing the next */ + uint32_t cyc_cmo_wait_before = cyc_cmo; + if (out_inflight) { + _t0 = perf_get_cycles(); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmo, _t0); + out_inflight = 0; + } + printf("Tile %u GEMM4: prev O write wait cycles: %u\n", + hartid, + cyc_cmo - cyc_cmo_wait_before); + + uint32_t cyc_cmo_issue_before = cyc_cmo; + /* Issue O batch write to L2 (non-blocking; overlaps with next compute) */ + _t0 = perf_get_cycles(); + idma_memcpy_1d(&idma_ctrl, + 1, + (uint32_t)o_out + row_index * DIM_F * 2, + obi_o_batch, + batch_rows * DIM_F * 2); + PERF_DELTA(cyc_cmo, _t0); + out_inflight = 1; + + total_o_done += batch_rows; + printf("Tile %u GEMM4: O write issue cycles: %u (total_o_done=%u/%u)\n", + hartid, + cyc_cmo - cyc_cmo_issue_before, + total_o_done, + num_rows); + } + + uint32_t cyc_cmo_drain_before = cyc_cmo; + /* Drain the final outstanding O write before the closing barrier */ + if (out_inflight) { + _t0 = perf_get_cycles(); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_cmo, _t0); + } + printf("Tile %u GEMM4: final O drain cycles: %u\n", + hartid, + cyc_cmo - cyc_cmo_drain_before); + } + } + + /* ------------------------------------------------------------------ */ + /* Final barrier: wait for all tiles to finish before validation */ + /* ------------------------------------------------------------------ */ + uint32_t cyc_barrier_final_before = cyc_barrier; + _t0 = perf_get_cycles(); + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + PERF_DELTA(cyc_barrier, _t0); + printf("Tile %u: final barrier cycles: %u\n", hartid, cyc_barrier - cyc_barrier_final_before); + + perf_stop(); + + /* Validation: tile 0 checks O against golden */ + uint32_t errors = 0; + + if (hartid == 0) { + for (uint32_t i = 0; i < DIM_A; i++) { + for (uint32_t j = 0; j < DIM_F; j++) { + float16 computed = *(volatile float16 *)(&o_out[i * DIM_F + j]); + float16 expected = o_golden[i * DIM_F + j]; + + uint16_t uc = *(uint16_t *)&computed; + uint16_t ue = *(uint16_t *)&expected; + + int32_t vc = fp16_to_millis(uc); + int32_t ve = fp16_to_millis(ue); + + int32_t abs_diff = vc - ve; + if (abs_diff < 0) + abs_diff = -abs_diff; + if (abs_diff > abs_threshold_millis) { + errors++; + printf("O[%d][%d]: got=%f (0x%x) exp=%f (0x%x) (abs_diff=%ld)\n", + i, + j, + fp16_to_f64(uc), + uc, + fp16_to_f64(ue), + ue, + (long)abs_diff); + } + } + } + + printf("\nTest complete. Errors: %d / %d\n\n", errors, DIM_A * DIM_F); + printf("Cycles: %u\n", perf_get_cycles()); + } + + /* Per-tile breakdown: print for one representative per GEMM group. + * GEMM1→tile 0, GEMM2→tile 16, GEMM3→tile 2, GEMM4→tile 20. + * Fields not used by a tile's role are zero. */ + if (hartid == 0 || hartid == 16 || hartid == 2 || hartid == 20) { + const char *role = (gemm1_idx >= 0) ? "GEMM1" + : (gemm2_idx >= 0) ? "GEMM2" + : (gemm3_idx >= 0) ? "GEMM3" + : (gemm4_idx >= 0) ? "GEMM4" + : "idle"; + printf("[tile %2u %s] barrier=%u cmi=%u compute=%u push=%u spin=%u cmo=%u\n", + hartid, + role, + cyc_barrier, + cyc_cmi, + cyc_compute, + cyc_push, + cyc_spin, + cyc_cmo); + } + + return errors; +} diff --git a/tests/magia/mesh/gemm_comm/gemm_comm_test.png b/tests/magia/mesh/gemm_comm/gemm_comm_test.png new file mode 100644 index 00000000..8421c912 Binary files /dev/null and b/tests/magia/mesh/gemm_comm/gemm_comm_test.png differ diff --git a/targets/magia_v2/include/utils/gemm_utils.h b/tests/magia/mesh/gemm_comm/gemm_utils.h similarity index 100% rename from targets/magia_v2/include/utils/gemm_utils.h rename to tests/magia/mesh/gemm_comm/gemm_utils.h diff --git a/tests/magia/mesh/gemm/via_l2/gen_golden.py b/tests/magia/mesh/gemm_comm/gen_golden.py similarity index 84% rename from tests/magia/mesh/gemm/via_l2/gen_golden.py rename to tests/magia/mesh/gemm_comm/gen_golden.py index 785ccd32..49a41f98 100644 --- a/tests/magia/mesh/gemm/via_l2/gen_golden.py +++ b/tests/magia/mesh/gemm_comm/gen_golden.py @@ -9,8 +9,8 @@ All GEMMs use fp16 throughout (inputs, accumulation, outputs). Usage: - python3 tests/magia/mesh/gemm/via_l2/gen_golden.py - python3 tests/magia/mesh/gemm/via_l2/gen_golden.py --dim-a 8 --dim-b 12 --dim-c 16 --dim-d 8 --dim-e 8 --dim-f 12 + python3 tests/magia/mesh/gemm_comm/gen_golden.py + python3 tests/magia/mesh/gemm_comm/gen_golden.py --dim-a 8 --dim-b 12 --dim-c 16 --dim-d 8 --dim-e 8 --dim-f 12 """ import argparse @@ -19,9 +19,13 @@ import torch -DEFAULT_OUTPUT = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "include", "test.h" -) +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +DEFAULT_OUTPUT = os.path.join(_SCRIPT_DIR, "via_L2", "naive", "include", "test.h") +INTERLACED_OUTPUT = os.path.join(_SCRIPT_DIR, "via_L2", "interlaced", "include", "test.h") +L1_NAIVE_OUTPUT = os.path.join(_SCRIPT_DIR, "via_L1", "naive", "include", "test.h") +L1_INTERLACED_OUTPUT = os.path.join(_SCRIPT_DIR, "via_L1", "interlaced", "include", "test.h") +FIFO_OUTPUT = os.path.join(_SCRIPT_DIR, "fifo", "include", "test.h") def parse_args(): @@ -203,7 +207,19 @@ def main(): write_test_h(args.output, (a, b, c, d, e, f), args.seed, m1, m2, m3, m4, m5, r1, r2, r3, o) - print(f"\nWrote {args.output}:") + # Also generate for the other variants (same golden data, different scheduling) + if args.output == DEFAULT_OUTPUT: + write_test_h(INTERLACED_OUTPUT, (a, b, c, d, e, f), args.seed, + m1, m2, m3, m4, m5, r1, r2, r3, o) + write_test_h(L1_NAIVE_OUTPUT, (a, b, c, d, e, f), args.seed, + m1, m2, m3, m4, m5, r1, r2, r3, o) + write_test_h(L1_INTERLACED_OUTPUT, (a, b, c, d, e, f), args.seed, + m1, m2, m3, m4, m5, r1, r2, r3, o) + write_test_h(FIFO_OUTPUT, (a, b, c, d, e, f), args.seed, + m1, m2, m3, m4, m5, r1, r2, r3, o) + print(f"\nWrote {args.output}, {INTERLACED_OUTPUT}, {L1_NAIVE_OUTPUT}, {L1_INTERLACED_OUTPUT}, and {FIFO_OUTPUT}:") + else: + print(f"\nWrote {args.output}:") print(f" m1_inp: {a*b} values ({a}x{b})") print(f" m2_inp: {b*c} values ({b}x{c})") print(f" m3_inp: {c*d} values ({c}x{d})") diff --git a/tests/magia/mesh/gemm_comm/via_L1/interlaced/CMakeLists.txt b/tests/magia/mesh/gemm_comm/via_L1/interlaced/CMakeLists.txt new file mode 100644 index 00000000..a5653926 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L1/interlaced/CMakeLists.txt @@ -0,0 +1,12 @@ +set(TEST_NAME test_gemm_l1_interlaced) + +file(GLOB_RECURSE TEST_SRCS "src/*.c") + +add_executable(${TEST_NAME} ${TEST_SRCS}) +target_include_directories(${TEST_NAME} PUBLIC include ${CMAKE_CURRENT_SOURCE_DIR}/../..) +target_compile_options(${TEST_NAME} PRIVATE -O2) +target_link_libraries(${TEST_NAME} PUBLIC runtime hal) + +add_custom_command( + TARGET ${TEST_NAME} POST_BUILD + COMMAND ${CMAKE_OBJDUMP} -dhS -Mmarch=${ISA} $ > $.s) diff --git a/tests/magia/mesh/gemm_comm/via_L1/interlaced/README.md b/tests/magia/mesh/gemm_comm/via_L1/interlaced/README.md new file mode 100644 index 00000000..b908beae --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L1/interlaced/README.md @@ -0,0 +1,103 @@ +# via_l1_interlaced — GEMM Chain Test (L1 communication with improved data movement) + +4-GEMM chain with task-level and row-parallel data parallelism across a 64-tile (8×8) mesh. +Inter-tile communication happens through L1 memory, in an interlaced way: tiles DMA intermediate results +directly into the L1 locations of tiles that need that data for subsequent computations. + +## Overview + +- Phase 1: + - Move data L2->L1 (GEMM 1, GEMM 2, and M5 for GEMM 4) + - GEMM 1 and GEMM 2 + - Move result source L1->target L1 (GEMM 1 and GEMM 2 to GEMM 3) + - === Local barrier === + +- Phase 2: + - GEMM 3 + - Move result source L1->target L1 (GEMM 3 to GEMM 4) + - === Global barrier === + +- Phase 3: + - GEMM 4 + - Move result L1->L2 (GEMM 4) + - === Global barrier === + +## Tile Groups + +| Group | Count | Role | +|---------|-------|---------------| +| GEMM 1 | 4 | Phase 1 | +| GEMM 2 | 24 | Phase 1 | +| GEMM 3 | 12 | Phase 2 | +| GEMM 4 | 24 | Phase 3 | + +## Phases + +### Phase 0 — Initialization +1. Each tile reads its `hartid`, derives `x_id`, `y_id`, and its L1 base address. +2. iDMA, RedMulE, FractalSync, and EventUnit controllers are initialized. +3. **Global barrier** (FractalSync at `MAX_SYNC_LVL - 1`): all tiles wait until every tile has + finished BSS zeroing (performed by tile 0 in `crt0.S`) before any tile writes to L2 output + buffers. + +--- + +### Phase 1 — Parallel GEMMs (GEMM1 ∥ GEMM2) + +Both groups run concurrently. Within each group, output rows are split evenly across tiles. + +**GEMM1** — `R1[A×C] = M1[A×B] @ M2[B×C]` + +For each tile in the group: +1. Compute the tile's row slice (`start_row`, `num_rows`) of the output. +2. DMA `M1[num_rows × B]` from L2 into L1. +3. DMA full `M2[B × C]` from L2 into L1. +4. Zero the L1 accumulator buffer for `R1_slice`. +5. RedMulE GEMM: `R1_slice = M1_slice @ M2`. +6. DMA `R1_slice` to correct positions in L1 of tiles responsible for GEMM 3, where R1 is needed. + +**GEMM2** — `R2[C×E] = M3[C×D] @ M4[D×E]` + +Same steps as GEMM1, operating on `M3`, `M4` → `r2_out` (`R2_slice = M3_slice @ M4`). + +**GEMM4** — `O[A×F] = R3[A×E] @ M5[E×F]` + +1. DMA full `M5[E × F]` from L2 into L1. + +**Local barrier** — Synchronize GEMM 1 and GEMM 2 responsible tiles before Phase 2 reads `r1_out` / `r2_out`. + +--- + +### Phase 2 — GEMM3 + +**GEMM3** — `R3[A×E] = R1[A×C] @ R2[C×E]` + +For each tile in the group: +1. Compute the tile's row slice of `R3`. +2. Zero the L1 accumulator buffer for `R3_slice`. +3. RedMulE GEMM: `R3_slice = R1_slice @ R2`. +4. DMA `R3_slice` to correct positions in L1 of tiles responsible for GEMM 4, where R1 is needed. + +**Global barrier** — all tiles synchronize before Phase 3 reads `r3_out`. + +--- + +### Phase 3 — GEMM4 + +**GEMM4** — `O[A×F] = R3[A×E] @ M5[E×F]` + +For each tile in the group: +1. Compute the tile's row slice of `O`. +3. Zero the L1 accumulator buffer for `O_slice`. +4. RedMulE GEMM: `O_slice = R3_slice @ M5`. +5. DMA `O_slice` back to L2 at `o_out[start_row * F]`. + +**Global barrier** — all tiles synchronize before validation. + +--- + +### Validation + +Tile 0 compares each element of `o_out` against the precomputed `o_golden` array. +Values are compared as integer millis (via `fp16_to_millis`) with an absolute threshold of +`0.008`. Errors and the final count are printed via `printf`. diff --git a/tests/magia/mesh/gemm_comm/via_L1/interlaced/src/test.c b/tests/magia/mesh/gemm_comm/via_L1/interlaced/src/test.c new file mode 100644 index 00000000..db689112 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L1/interlaced/src/test.c @@ -0,0 +1,450 @@ +// Copyright 2025 University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include "test.h" + +#include "tile.h" +#include "gemm_utils.h" +#include "fsync.h" +#include "idma.h" +#include "redmule.h" +#include "eventunit.h" + +#define WAIT_MODE WFE + +/** + * Tile group definitions for 8x8 mesh (64 tiles). + * + * GEMM1 tiles: [0, 1, 8, 9] (4 tiles, top-left 2x2 block) + * GEMM2 tiles: [16-19, 24-27, 32-35, + * 40-43, 48-51, 56-59] (24 tiles, rows 2-7 cols 0-3) + * GEMM3 tiles: [2-7, 10-15] (12 tiles, rows 0-1 cols 2-7) + * GEMM4 tiles: [20-23, 28-31, 36-39, + * 44-47, 52-55, 60-63] (24 tiles, rows 2-7 cols 4-7) + */ +#define GEMM1_N_TILES 4 +#define GEMM2_N_TILES 24 +#define GEMM3_N_TILES 12 +#define GEMM4_N_TILES 24 + +#define abs_threshold_millis 8 /* 0.008 expressed as integer millis */ + +static const uint32_t gemm1_tiles[GEMM1_N_TILES] = {0, 1, 8, 9}; + +static const uint32_t gemm2_tiles[GEMM2_N_TILES] = {16, 17, 18, 19, 24, 25, 26, 27, 32, 33, 34, 35, + 40, 41, 42, 43, 48, 49, 50, 51, 56, 57, 58, 59}; + +static const uint32_t gemm3_tiles[GEMM3_N_TILES] = {2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15}; + +static const uint32_t gemm4_tiles[GEMM4_N_TILES] = {20, 21, 22, 23, 28, 29, 30, 31, 36, 37, 38, 39, + 44, 45, 46, 47, 52, 53, 54, 55, 60, 61, 62, 63}; + +int get_local_idx(uint32_t hartid, const uint32_t *tiles, uint32_t n_tiles) +{ + for (uint32_t i = 0; i < n_tiles; i++) + if (tiles[i] == hartid) + return (int)i; + return -1; +} + +void get_row_range(uint32_t local_idx, + uint32_t n_tiles, + uint32_t total_rows, + uint32_t *start_row, + uint32_t *num_rows) +{ + uint32_t base = total_rows / n_tiles; + uint32_t rem = total_rows % n_tiles; + *start_row = local_idx * base + (local_idx < rem ? local_idx : rem); + *num_rows = base + (local_idx < rem ? 1 : 0); +} + +void mem_set_zero(uint32_t o, uint32_t dim) +{ + for (uint32_t i = 0; i < dim; i++) + mmio16(o + i * 2) = 0x0000; +} + +/** + * GEMM chain test with L1-to-L1 communication and interlaced data movement. + * + * Combines L1 direct communication (no L2 for intermediate results) with + * interlaced scheduling (GEMM4 tiles prefetch M5 during Phase 1). + * + * Phase 1 (parallel): + * GEMM1 group (4 tiles): R1[AxC] = M1[AxB] @ M2[BxC] + * -> scatter R1 rows to GEMM3 tiles' L1 + * GEMM2 group (24 tiles): R2[CxE] = M3[CxD] @ M4[DxE] + * -> scatter R2 rows to all GEMM3 tiles' L1 + * GEMM4 group (24 tiles): prefetch M5[ExF] into L1 + * + * Phase 2: + * GEMM3 group (12 tiles): R3[AxE] = R1[AxC] @ R2[CxE] + * (R1, R2 already in L1 from Phase 1 scatter) + * -> scatter R3 rows to GEMM4 tiles' L1 + * + * Phase 3: + * GEMM4 group (24 tiles): O[AxF] = R3[AxE] @ M5[ExF] + * (R3 already in L1 from Phase 2 scatter; M5 prefetched in Phase 1) + * -> write O back to L2 + * + * Requires 8x8 mesh (64 tiles). + */ +int main(void) +{ + /** + * 0. Initializations + */ + uint32_t hartid = get_hartid(); + uint32_t x_id = GET_X_ID(hartid); + uint32_t y_id = GET_Y_ID(hartid); + uint32_t l1_tile_base = get_l1_base(hartid); + + // Init iDMA + idma_config_t idma_cfg = {.hartid = hartid}; + idma_controller_t idma_ctrl = { + .base = NULL, + .cfg = &idma_cfg, + .api = &idma_api, + }; + idma_init(&idma_ctrl); + + // Init RedMulE + redmule_config_t redmule_cfg = {.hartid = hartid}; + redmule_controller_t redmule_ctrl = { + .base = NULL, + .cfg = &redmule_cfg, + .api = &redmule_api, + }; + redmule_init(&redmule_ctrl); + + // Init FractalSync + fsync_config_t fsync_cfg = {.hartid = hartid}; + fsync_controller_t fsync_ctrl = { + .base = NULL, + .cfg = &fsync_cfg, + .api = &fsync_api, + }; + fsync_init(&fsync_ctrl); + +// Init the Event Unit controller +#if STALLING == 0 + eu_config_t eu_cfg = {.hartid = hartid}; + eu_controller_t eu_ctrl = { + .base = NULL, + .cfg = &eu_cfg, + .api = &eu_api, + }; + + eu_init(&eu_ctrl); + eu_clear_events(0xFFFFFFFF); + eu_fsync_init(&eu_ctrl, 0); + eu_idma_init(&eu_ctrl, 0); + eu_redmule_init(&eu_ctrl, 0); +#endif + + // Determine each tile's group membership up front (needed for interlaced scheduling) + int gemm1_idx = get_local_idx(hartid, gemm1_tiles, GEMM1_N_TILES); + int gemm2_idx = get_local_idx(hartid, gemm2_tiles, GEMM2_N_TILES); + int gemm3_idx = get_local_idx(hartid, gemm3_tiles, GEMM3_N_TILES); + int gemm4_idx = get_local_idx(hartid, gemm4_tiles, GEMM4_N_TILES); + + // Global barrier: ensure all tiles have completed startup (including BSS zeroing + // in crt0.S) before any tile begins writing results to L2 output buffers. + // Without this, slow tiles still zeroing BSS can overwrite Phase 1 results. + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + perf_reset(); + perf_start(); + + /** + * Phase 1: GEMM1 and GEMM2 in parallel, GEMM4 prefetches M5 + * GEMM1 group: R1 = M1 @ M2 -> scatter R1 to GEMM3 tiles' L1 + * GEMM2 group: R2 = M3 @ M4 -> scatter R2 to GEMM3 tiles' L1 + * GEMM4 group: DMA M5 into L1 at Phase 3 offset + */ + if (gemm1_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm1_idx, GEMM1_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: M1_slice, M2, R1_slice + uint32_t obi_m1 = l1_tile_base; + uint32_t obi_m2 = obi_m1 + (num_rows * DIM_B * 2); + uint32_t obi_r1 = obi_m2 + (DIM_B * DIM_C * 2); + + // Load slice of M1 [num_rows x B] from L2 + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m1_inp + start_row * DIM_B * 2, + obi_m1, + num_rows * DIM_B * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Load full M2 [BxC] from L2 + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m2_inp, obi_m2, DIM_B * DIM_C * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Zero accumulator and compute: R1_slice = M1_slice @ M2 + mem_set_zero(obi_r1, num_rows * DIM_C); + redmule_gemm(&redmule_ctrl, + obi_m1, + obi_m2, + obi_r1, + (uint16_t)num_rows, + (uint16_t)DIM_B, + (uint16_t)DIM_C); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Scatter R1 rows directly to GEMM3 tiles' L1. + // GEMM3 tile j's L1 layout: [R1_slice | R2 | R3_slice] + for (uint32_t j = 0; j < GEMM3_N_TILES; j++) { + uint32_t g3_start, g3_nrows; + get_row_range(j, GEMM3_N_TILES, DIM_A, &g3_start, &g3_nrows); + if (g3_nrows == 0) + continue; + + // Overlap between this tile's R1 rows and GEMM3 tile j's needed rows + uint32_t ov_start = start_row > g3_start ? start_row : g3_start; + uint32_t ov_end_1 = start_row + num_rows; + uint32_t ov_end_3 = g3_start + g3_nrows; + uint32_t ov_end = ov_end_1 < ov_end_3 ? ov_end_1 : ov_end_3; + + if (ov_start >= ov_end) + continue; + uint32_t ov_count = ov_end - ov_start; + + uint32_t g3_l1 = get_l1_base(gemm3_tiles[j]); + uint32_t src_off = (ov_start - start_row) * DIM_C * 2; + uint32_t dst_off = (ov_start - g3_start) * DIM_C * 2; + + idma_memcpy_1d( + &idma_ctrl, 1, g3_l1 + dst_off, obi_r1 + src_off, ov_count * DIM_C * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + } + + if (gemm2_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm2_idx, GEMM2_N_TILES, DIM_C, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: M3_slice, M4, R2_slice + uint32_t obi_m3 = l1_tile_base; + uint32_t obi_m4 = obi_m3 + (num_rows * DIM_D * 2); + uint32_t obi_r2 = obi_m4 + (DIM_D * DIM_E * 2); + + // Load slice of M3 [num_rows x D] from L2 + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m3_inp + start_row * DIM_D * 2, + obi_m3, + num_rows * DIM_D * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Load full M4 [DxE] from L2 + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m4_inp, obi_m4, DIM_D * DIM_E * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Zero accumulator and compute: R2_slice = M3_slice @ M4 + mem_set_zero(obi_r2, num_rows * DIM_E); + redmule_gemm(&redmule_ctrl, + obi_m3, + obi_m4, + obi_r2, + (uint16_t)num_rows, + (uint16_t)DIM_D, + (uint16_t)DIM_E); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Scatter R2 rows to every GEMM3 tile's L1. + // Each GEMM3 tile needs the full R2, so every GEMM2 tile sends + // its slice to all GEMM3 tiles at the correct row offset. + // GEMM3 tile j's L1 layout: [R1_slice: g3_nrows*C*2 | R2: C*E*2 | ...] + for (uint32_t j = 0; j < GEMM3_N_TILES; j++) { + uint32_t g3_start, g3_nrows; + get_row_range(j, GEMM3_N_TILES, DIM_A, &g3_start, &g3_nrows); + if (g3_nrows == 0) + continue; + + uint32_t g3_l1 = get_l1_base(gemm3_tiles[j]); + uint32_t r2_base = g3_l1 + g3_nrows * DIM_C * 2; + uint32_t dst = r2_base + start_row * DIM_E * 2; + + idma_memcpy_1d(&idma_ctrl, 1, dst, obi_r2, num_rows * DIM_E * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + } + + if (gemm4_idx >= 0) { + // Prefetch M5 into L1 during Phase 1, placed at Phase 3 offset. + // Phase 3 L1 layout: R3_slice | M5 | O_slice + uint32_t start_row, num_rows; + get_row_range(gemm4_idx, GEMM4_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + uint32_t obi_m5 = l1_tile_base + (num_rows * DIM_E * 2); + + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m5_inp, obi_m5, DIM_E * DIM_F * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + } + } + + // Local barrier: wait for Phase 1 to complete before GEMM3 reads r1_out / r2_out. + // TODO: use a lower sync level once a safe sub-global level is determined + // for the current tile topology. + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + /** + * Phase 2: GEMM3 (row-parallel) + * GEMM3 group: R3 = R1 @ R2 + * R1 and R2 are already in this tile's L1 (placed by Phase 1 scatter). + * After computing, scatter R3 rows to GEMM4 tiles' L1. + */ + if (gemm3_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm3_idx, GEMM3_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout (populated by Phase 1): [R1_slice | R2 | R3_slice] + uint32_t obi_r1 = l1_tile_base; + uint32_t obi_r2 = obi_r1 + (num_rows * DIM_C * 2); + uint32_t obi_r3 = obi_r2 + (DIM_C * DIM_E * 2); + + // Zero accumulator and compute: R3_slice = R1_slice @ R2 + mem_set_zero(obi_r3, num_rows * DIM_E); + redmule_gemm(&redmule_ctrl, + obi_r1, + obi_r2, + obi_r3, + (uint16_t)num_rows, + (uint16_t)DIM_C, + (uint16_t)DIM_E); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Scatter R3 rows to GEMM4 tiles' L1. + // GEMM4 tile k's L1 layout: [R3_slice | M5 | O_slice] + for (uint32_t k = 0; k < GEMM4_N_TILES; k++) { + uint32_t g4_start, g4_nrows; + get_row_range(k, GEMM4_N_TILES, DIM_A, &g4_start, &g4_nrows); + if (g4_nrows == 0) + continue; + + // Overlap between this tile's R3 rows and GEMM4 tile k's needed rows + uint32_t ov_start = start_row > g4_start ? start_row : g4_start; + uint32_t ov_end_3 = start_row + num_rows; + uint32_t ov_end_4 = g4_start + g4_nrows; + uint32_t ov_end = ov_end_3 < ov_end_4 ? ov_end_3 : ov_end_4; + + if (ov_start >= ov_end) + continue; + uint32_t ov_count = ov_end - ov_start; + + uint32_t g4_l1 = get_l1_base(gemm4_tiles[k]); + uint32_t src_off = (ov_start - start_row) * DIM_E * 2; + uint32_t dst_off = (ov_start - g4_start) * DIM_E * 2; + + idma_memcpy_1d( + &idma_ctrl, 1, g4_l1 + dst_off, obi_r3 + src_off, ov_count * DIM_E * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + } + + // Global barrier: wait for Phase 2 to complete + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + /** + * Phase 3: GEMM4 (row-parallel) — M5 already in L1 from Phase 1 prefetch + * GEMM4 group: O = R3 @ M5 + * R3 is already in this tile's L1 (placed by Phase 2 scatter). + * M5 was prefetched in Phase 1. O is written back to L2. + */ + if (gemm4_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm4_idx, GEMM4_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: R3_slice (from Phase 2 scatter) | M5 (prefetched in Phase 1) | O_slice + uint32_t obi_r3 = l1_tile_base; + uint32_t obi_m5 = obi_r3 + (num_rows * DIM_E * 2); + uint32_t obi_o = obi_m5 + (DIM_E * DIM_F * 2); + + // R3 already in L1 — no DMA needed + // M5 already in L1 at obi_m5 — no DMA needed + + // Zero accumulator and compute: O_slice = R3_slice @ M5 + mem_set_zero(obi_o, num_rows * DIM_F); + redmule_gemm(&redmule_ctrl, + obi_r3, + obi_m5, + obi_o, + (uint16_t)num_rows, + (uint16_t)DIM_E, + (uint16_t)DIM_F); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Write O slice back to L2 at correct offset + idma_memcpy_1d(&idma_ctrl, + 1, + (uint32_t)o_out + start_row * DIM_F * 2, + obi_o, + num_rows * DIM_F * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + + // Global barrier: wait for Phase 3 to complete + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + perf_stop(); + + /** + * Validation: Tile 0 checks O against golden + */ + uint32_t errors = 0; + + if (hartid == 0) { + for (uint32_t i = 0; i < DIM_A; i++) { + for (uint32_t j = 0; j < DIM_F; j++) { + float16 computed = *(volatile float16 *)(&o_out[i * DIM_F + j]); + float16 expected = o_golden[i * DIM_F + j]; + + uint16_t uc = *(uint16_t *)&computed; + uint16_t ue = *(uint16_t *)&expected; + + int32_t vc = fp16_to_millis(uc); + int32_t ve = fp16_to_millis(ue); + + int32_t abs_diff = vc - ve; + if (abs_diff < 0) + abs_diff = -abs_diff; + if (abs_diff > abs_threshold_millis) { + errors++; + + printf("O[%d][%d]: got=%f (0x%x) exp=%f (0x%x) (abs_diff=%ld)\n", + i, + j, + fp16_to_f64(uc), + uc, + fp16_to_f64(ue), + ue, + (long)abs_diff); + } + } + } + + printf("\nTest complete. Errors: %d / %d\n\n", errors, DIM_A * DIM_F); + printf("Cycles: %u\n", perf_get_cycles()); + } + + return errors; +} diff --git a/tests/magia/mesh/gemm_comm/via_L1/naive/CMakeLists.txt b/tests/magia/mesh/gemm_comm/via_L1/naive/CMakeLists.txt new file mode 100644 index 00000000..44f59247 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L1/naive/CMakeLists.txt @@ -0,0 +1,12 @@ +set(TEST_NAME test_gemm_l1_naive) + +file(GLOB_RECURSE TEST_SRCS "src/*.c") + +add_executable(${TEST_NAME} ${TEST_SRCS}) +target_include_directories(${TEST_NAME} PUBLIC include ${CMAKE_CURRENT_SOURCE_DIR}/../..) +target_compile_options(${TEST_NAME} PRIVATE -O2) +target_link_libraries(${TEST_NAME} PUBLIC runtime hal) + +add_custom_command( + TARGET ${TEST_NAME} POST_BUILD + COMMAND ${CMAKE_OBJDUMP} -dhS -Mmarch=${ISA} $ > $.s) diff --git a/tests/magia/mesh/gemm_comm/via_L1/naive/README.md b/tests/magia/mesh/gemm_comm/via_L1/naive/README.md new file mode 100644 index 00000000..ff46eed9 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L1/naive/README.md @@ -0,0 +1,101 @@ +# via_l1_naive — GEMM Chain Test (Direct L1-to-L1 inter-mesh communication) + +4-GEMM chain with task-level and row-parallel data parallelism across a 64-tile (8×8) mesh. +Inter-tile communication happens through L1 memory: tiles DMA intermediate results +directly into the L1 locations of tiles that need that data for subsequent computations. + +## Overview + +- Phase 1: + - Move data L2->L1 (GEMM 1 and GEMM 2) + - GEMM 1 and GEMM 2 + - Move result source L1->target L1 (GEMM 1 and GEMM 2 to GEMM 3) + - === Global barrier === + +- Phase 2: + - GEMM 3 + - Move result source L1->target L1 (GEMM 3 to GEMM 4) + - === Global barrier === + +- Phase 3: + - Move data L2->L1 (M5 for GEMM 4) + - GEMM 4 + - Move result L1->L2 (GEMM 4) + - === Global barrier === + +## Tile Groups + +| Group | Count | Role | +|---------|-------|---------------| +| GEMM 1 | 4 | Phase 1 | +| GEMM 2 | 24 | Phase 1 | +| GEMM 3 | 12 | Phase 2 | +| GEMM 4 | 24 | Phase 3 | + +## Phases + +### Phase 0 — Initialization +1. Each tile reads its `hartid`, derives `x_id`, `y_id`, and its L1 base address. +2. iDMA, RedMulE, FractalSync, and EventUnit controllers are initialized. +3. **Global barrier** (FractalSync at `MAX_SYNC_LVL - 1`): all tiles wait until every tile has + finished BSS zeroing (performed by tile 0 in `crt0.S`) before any tile writes to L2 output + buffers. + +--- + +### Phase 1 — Parallel GEMMs (GEMM1 ∥ GEMM2) + +Both groups run concurrently. Within each group, output rows are split evenly across tiles. + +**GEMM1** — `R1[A×C] = M1[A×B] @ M2[B×C]` + +For each tile in the group: +1. Compute the tile's row slice (`start_row`, `num_rows`) of the output. +2. DMA `M1[num_rows × B]` from L2 into L1. +3. DMA full `M2[B × C]` from L2 into L1. +4. Zero the L1 accumulator buffer for `R1_slice`. +5. RedMulE GEMM: `R1_slice = M1_slice @ M2`. +6. DMA `R1_slice` to correct positions in L1 of tiles responsible for GEMM 3, where R1 is needed. + +**GEMM2** — `R2[C×E] = M3[C×D] @ M4[D×E]` + +Same steps as GEMM1, operating on `M3`, `M4` → `r2_out` (`R2_slice = M3_slice @ M4`). + +**Global barrier** — all tiles synchronize before Phase 2 reads `r1_out` / `r2_out`. + +--- + +### Phase 2 — GEMM3 + +**GEMM3** — `R3[A×E] = R1[A×C] @ R2[C×E]` + +For each tile in the group: +1. Compute the tile's row slice of `R3`. +2. Zero the L1 accumulator buffer for `R3_slice`. +3. RedMulE GEMM: `R3_slice = R1_slice @ R2`. +4. DMA `R3_slice` to correct positions in L1 of tiles responsible for GEMM 4, where R1 is needed. + +**Global barrier** — all tiles synchronize before Phase 3 reads `r3_out`. + +--- + +### Phase 3 — GEMM4 + +**GEMM4** — `O[A×F] = R3[A×E] @ M5[E×F]` + +For each tile in the group: +1. Compute the tile's row slice of `O`. +2. DMA full `M5[E × F]` from L2 into L1. +3. Zero the L1 accumulator buffer for `O_slice`. +4. RedMulE GEMM: `O_slice = R3_slice @ M5`. +5. DMA `O_slice` back to L2 at `o_out[start_row * F]`. + +**Global barrier** — all tiles synchronize before validation. + +--- + +### Validation + +Tile 0 compares each element of `o_out` against the precomputed `o_golden` array. +Values are compared as integer millis (via `fp16_to_millis`) with an absolute threshold of +`0.008`. Errors and the final count are printed via `printf`. diff --git a/tests/magia/mesh/gemm_comm/via_L1/naive/src/test.c b/tests/magia/mesh/gemm_comm/via_L1/naive/src/test.c new file mode 100644 index 00000000..401c1341 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L1/naive/src/test.c @@ -0,0 +1,433 @@ +// Copyright 2025 University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include "test.h" + +#include "tile.h" +#include "gemm_utils.h" +#include "fsync.h" +#include "idma.h" +#include "redmule.h" +#include "eventunit.h" + +#define WAIT_MODE WFE + +/** + * Tile group definitions for 8x8 mesh (64 tiles). + * + * GEMM1 tiles: [0, 1, 8, 9] (4 tiles, top-left 2x2 block) + * GEMM2 tiles: [16-19, 24-27, 32-35, + * 40-43, 48-51, 56-59] (24 tiles, rows 2-7 cols 0-3) + * GEMM3 tiles: [2-7, 10-15] (12 tiles, rows 0-1 cols 2-7) + * GEMM4 tiles: [20-23, 28-31, 36-39, + * 44-47, 52-55, 60-63] (24 tiles, rows 2-7 cols 4-7) + */ +#define GEMM1_N_TILES 4 +#define GEMM2_N_TILES 24 +#define GEMM3_N_TILES 12 +#define GEMM4_N_TILES 24 + +#define abs_threshold_millis 8 /* 0.008 expressed as integer millis */ + +static const uint32_t gemm1_tiles[GEMM1_N_TILES] = {0, 1, 8, 9}; + +static const uint32_t gemm2_tiles[GEMM2_N_TILES] = {16, 17, 18, 19, 24, 25, 26, 27, 32, 33, 34, 35, + 40, 41, 42, 43, 48, 49, 50, 51, 56, 57, 58, 59}; + +static const uint32_t gemm3_tiles[GEMM3_N_TILES] = {2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15}; + +static const uint32_t gemm4_tiles[GEMM4_N_TILES] = {20, 21, 22, 23, 28, 29, 30, 31, 36, 37, 38, 39, + 44, 45, 46, 47, 52, 53, 54, 55, 60, 61, 62, 63}; + +int get_local_idx(uint32_t hartid, const uint32_t *tiles, uint32_t n_tiles) +{ + for (uint32_t i = 0; i < n_tiles; i++) + if (tiles[i] == hartid) + return (int)i; + return -1; +} + +void get_row_range(uint32_t local_idx, + uint32_t n_tiles, + uint32_t total_rows, + uint32_t *start_row, + uint32_t *num_rows) +{ + uint32_t base = total_rows / n_tiles; + uint32_t rem = total_rows % n_tiles; + *start_row = local_idx * base + (local_idx < rem ? local_idx : rem); + *num_rows = base + (local_idx < rem ? 1 : 0); +} + +void mem_set_zero(uint32_t o, uint32_t dim) +{ + for (uint32_t i = 0; i < dim; i++) + mmio16(o + i * 2) = 0x0000; +} + +/** + * GEMM chain test with L1-to-L1 communication. + * + * Same 4-GEMM chain as the L2 naive variant, but intermediate results + * are DMA'd directly between tiles' L1 memories instead of going through L2. + * Only initial inputs (M1-M5) are loaded from L2, and the final output (O) + * is written back to L2 for validation. + * + * Phase 1 (parallel): + * GEMM1 group (4 tiles): R1[AxC] = M1[AxB] @ M2[BxC] + * → scatter R1 rows to GEMM3 tiles' L1 + * GEMM2 group (24 tiles): R2[CxE] = M3[CxD] @ M4[DxE] + * → scatter R2 rows to all GEMM3 tiles' L1 + * + * Phase 2: + * GEMM3 group (12 tiles): R3[AxE] = R1[AxC] @ R2[CxE] + * (R1, R2 already in L1 from Phase 1 scatter) + * → scatter R3 rows to GEMM4 tiles' L1 + * + * Phase 3: + * GEMM4 group (24 tiles): O[AxF] = R3[AxE] @ M5[ExF] + * (R3 already in L1 from Phase 2 scatter; M5 loaded from L2) + * → write O back to L2 + * + * Requires 8x8 mesh (64 tiles). + */ +int main(void) +{ + /** + * 0. Initializations + */ + uint32_t hartid = get_hartid(); + uint32_t x_id = GET_X_ID(hartid); + uint32_t y_id = GET_Y_ID(hartid); + uint32_t l1_tile_base = get_l1_base(hartid); + + // Init iDMA + idma_config_t idma_cfg = {.hartid = hartid}; + idma_controller_t idma_ctrl = { + .base = NULL, + .cfg = &idma_cfg, + .api = &idma_api, + }; + idma_init(&idma_ctrl); + + // Init RedMulE + redmule_config_t redmule_cfg = {.hartid = hartid}; + redmule_controller_t redmule_ctrl = { + .base = NULL, + .cfg = &redmule_cfg, + .api = &redmule_api, + }; + redmule_init(&redmule_ctrl); + + // Init FractalSync + fsync_config_t fsync_cfg = {.hartid = hartid}; + fsync_controller_t fsync_ctrl = { + .base = NULL, + .cfg = &fsync_cfg, + .api = &fsync_api, + }; + fsync_init(&fsync_ctrl); + +// Init the Event Unit controller +#if STALLING == 0 + eu_config_t eu_cfg = {.hartid = hartid}; + eu_controller_t eu_ctrl = { + .base = NULL, + .cfg = &eu_cfg, + .api = &eu_api, + }; + + eu_init(&eu_ctrl); + eu_clear_events(0xFFFFFFFF); + eu_fsync_init(&eu_ctrl, 0); + eu_idma_init(&eu_ctrl, 0); + eu_redmule_init(&eu_ctrl, 0); +#endif + + // Global barrier: ensure all tiles have completed startup (including BSS zeroing + // in crt0.S) before any tile begins writing results to L2 output buffers. + // Without this, slow tiles still zeroing BSS can overwrite Phase 1 results. + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + perf_reset(); + perf_start(); + + /** + * Phase 1: GEMM1 and GEMM2 in parallel (row-parallel within each group) + * GEMM1 group: R1 = M1 @ M2 → scatter R1 to GEMM3 tiles' L1 + * GEMM2 group: R2 = M3 @ M4 → scatter R2 to GEMM3 tiles' L1 + */ + int gemm1_idx = get_local_idx(hartid, gemm1_tiles, GEMM1_N_TILES); + if (gemm1_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm1_idx, GEMM1_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: M1_slice, M2, R1_slice + uint32_t obi_m1 = l1_tile_base; + uint32_t obi_m2 = obi_m1 + (num_rows * DIM_B * 2); + uint32_t obi_r1 = obi_m2 + (DIM_B * DIM_C * 2); + + // Load slice of M1 [num_rows x B] from L2 + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m1_inp + start_row * DIM_B * 2, + obi_m1, + num_rows * DIM_B * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Load full M2 [BxC] from L2 + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m2_inp, obi_m2, DIM_B * DIM_C * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Zero accumulator and compute: R1_slice = M1_slice @ M2 + mem_set_zero(obi_r1, num_rows * DIM_C); + redmule_gemm(&redmule_ctrl, + obi_m1, + obi_m2, + obi_r1, + (uint16_t)num_rows, + (uint16_t)DIM_B, + (uint16_t)DIM_C); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Scatter R1 rows directly to GEMM3 tiles' L1. + // GEMM3 tile j's L1 layout: [R1_slice | R2 | R3_slice] + for (uint32_t j = 0; j < GEMM3_N_TILES; j++) { + uint32_t g3_start, g3_nrows; + get_row_range(j, GEMM3_N_TILES, DIM_A, &g3_start, &g3_nrows); + if (g3_nrows == 0) + continue; + + // Overlap between this tile's R1 rows and GEMM3 tile j's needed rows + uint32_t ov_start = start_row > g3_start ? start_row : g3_start; + uint32_t ov_end_1 = start_row + num_rows; + uint32_t ov_end_3 = g3_start + g3_nrows; + uint32_t ov_end = ov_end_1 < ov_end_3 ? ov_end_1 : ov_end_3; + + if (ov_start >= ov_end) + continue; + uint32_t ov_count = ov_end - ov_start; + + uint32_t g3_l1 = get_l1_base(gemm3_tiles[j]); + uint32_t src_off = (ov_start - start_row) * DIM_C * 2; + uint32_t dst_off = (ov_start - g3_start) * DIM_C * 2; + + idma_memcpy_1d( + &idma_ctrl, 1, g3_l1 + dst_off, obi_r1 + src_off, ov_count * DIM_C * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + } + + int gemm2_idx = get_local_idx(hartid, gemm2_tiles, GEMM2_N_TILES); + if (gemm2_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm2_idx, GEMM2_N_TILES, DIM_C, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: M3_slice, M4, R2_slice + uint32_t obi_m3 = l1_tile_base; + uint32_t obi_m4 = obi_m3 + (num_rows * DIM_D * 2); + uint32_t obi_r2 = obi_m4 + (DIM_D * DIM_E * 2); + + // Load slice of M3 [num_rows x D] from L2 + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m3_inp + start_row * DIM_D * 2, + obi_m3, + num_rows * DIM_D * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Load full M4 [DxE] from L2 + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m4_inp, obi_m4, DIM_D * DIM_E * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Zero accumulator and compute: R2_slice = M3_slice @ M4 + mem_set_zero(obi_r2, num_rows * DIM_E); + redmule_gemm(&redmule_ctrl, + obi_m3, + obi_m4, + obi_r2, + (uint16_t)num_rows, + (uint16_t)DIM_D, + (uint16_t)DIM_E); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Scatter R2 rows to every GEMM3 tile's L1. + // Each GEMM3 tile needs the full R2, so every GEMM2 tile sends + // its slice to all GEMM3 tiles at the correct row offset. + // GEMM3 tile j's L1 layout: [R1_slice: g3_nrows*C*2 | R2: C*E*2 | ...] + for (uint32_t j = 0; j < GEMM3_N_TILES; j++) { + uint32_t g3_start, g3_nrows; + get_row_range(j, GEMM3_N_TILES, DIM_A, &g3_start, &g3_nrows); + if (g3_nrows == 0) + continue; + + uint32_t g3_l1 = get_l1_base(gemm3_tiles[j]); + uint32_t r2_base = g3_l1 + g3_nrows * DIM_C * 2; + uint32_t dst = r2_base + start_row * DIM_E * 2; + + idma_memcpy_1d(&idma_ctrl, 1, dst, obi_r2, num_rows * DIM_E * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + } + + // Global barrier: wait for Phase 1 to complete + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + /** + * Phase 2: GEMM3 (row-parallel) + * GEMM3 group: R3 = R1 @ R2 + * R1 and R2 are already in this tile's L1 (placed by Phase 1 scatter). + * After computing, scatter R3 rows to GEMM4 tiles' L1. + */ + int gemm3_idx = get_local_idx(hartid, gemm3_tiles, GEMM3_N_TILES); + if (gemm3_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm3_idx, GEMM3_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout (populated by Phase 1): [R1_slice | R2 | R3_slice] + uint32_t obi_r1 = l1_tile_base; + uint32_t obi_r2 = obi_r1 + (num_rows * DIM_C * 2); + uint32_t obi_r3 = obi_r2 + (DIM_C * DIM_E * 2); + + // Zero accumulator and compute: R3_slice = R1_slice @ R2 + mem_set_zero(obi_r3, num_rows * DIM_E); + redmule_gemm(&redmule_ctrl, + obi_r1, + obi_r2, + obi_r3, + (uint16_t)num_rows, + (uint16_t)DIM_C, + (uint16_t)DIM_E); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Scatter R3 rows to GEMM4 tiles' L1. + // GEMM4 tile k's L1 layout: [R3_slice | M5 | O_slice] + for (uint32_t k = 0; k < GEMM4_N_TILES; k++) { + uint32_t g4_start, g4_nrows; + get_row_range(k, GEMM4_N_TILES, DIM_A, &g4_start, &g4_nrows); + if (g4_nrows == 0) + continue; + + // Overlap between this tile's R3 rows and GEMM4 tile k's needed rows + uint32_t ov_start = start_row > g4_start ? start_row : g4_start; + uint32_t ov_end_3 = start_row + num_rows; + uint32_t ov_end_4 = g4_start + g4_nrows; + uint32_t ov_end = ov_end_3 < ov_end_4 ? ov_end_3 : ov_end_4; + + if (ov_start >= ov_end) + continue; + uint32_t ov_count = ov_end - ov_start; + + uint32_t g4_l1 = get_l1_base(gemm4_tiles[k]); + uint32_t src_off = (ov_start - start_row) * DIM_E * 2; + uint32_t dst_off = (ov_start - g4_start) * DIM_E * 2; + + idma_memcpy_1d( + &idma_ctrl, 1, g4_l1 + dst_off, obi_r3 + src_off, ov_count * DIM_E * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + } + + // Global barrier: wait for Phase 2 to complete + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + /** + * Phase 3: GEMM4 (row-parallel) + * GEMM4 group: O = R3 @ M5 + * R3 is already in this tile's L1 (placed by Phase 2 scatter). + * M5 is loaded from L2. O is written back to L2. + */ + int gemm4_idx = get_local_idx(hartid, gemm4_tiles, GEMM4_N_TILES); + if (gemm4_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm4_idx, GEMM4_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: [R3_slice (already here) | M5 | O_slice] + uint32_t obi_r3 = l1_tile_base; + uint32_t obi_m5 = obi_r3 + (num_rows * DIM_E * 2); + uint32_t obi_o = obi_m5 + (DIM_E * DIM_F * 2); + + // Load full M5 [ExF] from L2 + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m5_inp, obi_m5, DIM_E * DIM_F * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Zero accumulator and compute: O_slice = R3_slice @ M5 + mem_set_zero(obi_o, num_rows * DIM_F); + redmule_gemm(&redmule_ctrl, + obi_r3, + obi_m5, + obi_o, + (uint16_t)num_rows, + (uint16_t)DIM_E, + (uint16_t)DIM_F); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Write O slice back to L2 at correct offset + idma_memcpy_1d(&idma_ctrl, + 1, + (uint32_t)o_out + start_row * DIM_F * 2, + obi_o, + num_rows * DIM_F * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + + // Global barrier: wait for Phase 3 to complete + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + perf_stop(); + + /** + * Validation: Tile 0 checks O against golden + */ + uint32_t errors = 0; + + if (hartid == 0) { + for (uint32_t i = 0; i < DIM_A; i++) { + for (uint32_t j = 0; j < DIM_F; j++) { + float16 computed = *(volatile float16 *)(&o_out[i * DIM_F + j]); + float16 expected = o_golden[i * DIM_F + j]; + + uint16_t uc = *(uint16_t *)&computed; + uint16_t ue = *(uint16_t *)&expected; + + int32_t vc = fp16_to_millis(uc); + int32_t ve = fp16_to_millis(ue); + + int32_t abs_diff = vc - ve; + if (abs_diff < 0) + abs_diff = -abs_diff; + if (abs_diff > abs_threshold_millis) { + errors++; + + printf("O[%d][%d]: got=%f (0x%x) exp=%f (0x%x) (abs_diff=%ld)\n", + i, + j, + fp16_to_f64(uc), + uc, + fp16_to_f64(ue), + ue, + (long)abs_diff); + } + } + } + + printf("\nTest complete. Errors: %d / %d\n\n", errors, DIM_A * DIM_F); + printf("Cycles: %u\n", perf_get_cycles()); + } + + return errors; +} diff --git a/tests/magia/mesh/gemm_comm/via_L2/interlaced/CMakeLists.txt b/tests/magia/mesh/gemm_comm/via_L2/interlaced/CMakeLists.txt new file mode 100644 index 00000000..7a7d2d8f --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L2/interlaced/CMakeLists.txt @@ -0,0 +1,12 @@ +set(TEST_NAME test_gemm_l2_interlaced) + +file(GLOB_RECURSE TEST_SRCS "src/*.c") + +add_executable(${TEST_NAME} ${TEST_SRCS}) +target_include_directories(${TEST_NAME} PUBLIC include ${CMAKE_CURRENT_SOURCE_DIR}/../..) +target_compile_options(${TEST_NAME} PRIVATE -O2) +target_link_libraries(${TEST_NAME} PUBLIC runtime hal) + +add_custom_command( + TARGET ${TEST_NAME} POST_BUILD + COMMAND ${CMAKE_OBJDUMP} -dhS -Mmarch=${ISA} $ > $.s) diff --git a/tests/magia/mesh/gemm_comm/via_L2/interlaced/README.md b/tests/magia/mesh/gemm_comm/via_L2/interlaced/README.md new file mode 100644 index 00000000..c07e3bf6 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L2/interlaced/README.md @@ -0,0 +1,108 @@ +# via_l2_interlaced — GEMM Chain Test (L2 communication with improved data movement) + +4-GEMM chain with task-level and row-parallel data parallelism across a 64-tile (8×8) mesh. +Inter-tile communication happens entirely through shared L2 memory, in an interlaced way: tiles DMA intermediate results +into L2 buffers that subsequent phases read back. + +## Overview + +- Phase 1: + - Move data L2->L1 (GEMM 1, GEMM 2, and M5 for GEMM 4) + - GEMM 1 and GEMM 2 + - Move result L1->L2 (GEMM 1 and GEMM 2) + - === Local barrier === + +- Phase 2: + - Move data L2->L1 (GEMM 3) + - GEMM 3 + - Move result L1->L2 (GEMM 3) + - === Global barrier === + +- Phase 3: + - Move data L2->L1 (R3 for GEMM 4) + - GEMM 4 + - Move result L1->L2 (GEMM 4) + - === Global barrier === + +## Tile Groups + +| Group | Count | Role | +|---------|-------|---------------| +| GEMM 1 | 4 | Phase 1 | +| GEMM 2 | 24 | Phase 1 | +| GEMM 3 | 12 | Phase 2 | +| GEMM 4 | 24 | Phase 3 | + +## Phases + +### Phase 0 — Initialization +1. Each tile reads its `hartid`, derives `x_id`, `y_id`, and its L1 base address. +2. iDMA, RedMulE, FractalSync, and EventUnit controllers are initialized. +3. **Global barrier** (FractalSync at `MAX_SYNC_LVL - 1`): all tiles wait until every tile has + finished BSS zeroing (performed by tile 0 in `crt0.S`) before any tile writes to L2 output + buffers. + +--- + +### Phase 1 — Parallel GEMMs (GEMM1 ∥ GEMM2) + +Both groups run concurrently. Within each group, output rows are split evenly across tiles. + +**GEMM1** — `R1[A×C] = M1[A×B] @ M2[B×C]` + +For each tile in the group: +1. Compute the tile's row slice (`start_row`, `num_rows`) of the output. +2. DMA `M1[num_rows × B]` from L2 into L1. +3. DMA full `M2[B × C]` from L2 into L1. +4. Zero the L1 accumulator buffer for `R1_slice`. +5. RedMulE GEMM: `R1_slice = M1_slice @ M2`. +6. DMA `R1_slice` back to L2 at `r1_out[start_row * C]`. + +**GEMM2** — `R2[C×E] = M3[C×D] @ M4[D×E]` + +Same steps as GEMM1, operating on `M3`, `M4` → `r2_out` (`R2_slice = M3_slice @ M4`). + +**GEMM4** — `O[A×F] = R3[A×E] @ M5[E×F]` + +1. DMA full `M5[E × F]` from L2 into L1. + +**Local barrier** — Synchronize GEMM 1 and GEMM 2 responsible tiles before Phase 2 reads `r1_out` / `r2_out`. + +--- + +### Phase 2 — GEMM3 + +**GEMM3** — `R3[A×E] = R1[A×C] @ R2[C×E]` + +For each tile in the group: +1. Compute the tile's row slice of `R3`. +2. DMA `R1[num_rows × C]` from L2 (`r1_out`) into L1. +3. DMA full `R2[C × E]` from L2 (`r2_out`) into L1. +4. Zero the L1 accumulator buffer for `R3_slice`. +5. RedMulE GEMM: `R3_slice = R1_slice @ R2`. +6. DMA `R3_slice` back to L2 at `r3_out[start_row * E]`. + +**Global barrier** — all tiles synchronize before Phase 3 reads `r3_out`. + +--- + +### Phase 3 — GEMM4 + +**GEMM4** — `O[A×F] = R3[A×E] @ M5[E×F]` + +For each tile in the group: +1. Compute the tile's row slice of `O`. +2. DMA `R3[num_rows × E]` from L2 (`r3_out`) into L1. +3. Zero the L1 accumulator buffer for `O_slice`. +4. RedMulE GEMM: `O_slice = R3_slice @ M5`. +5. DMA `O_slice` back to L2 at `o_out[start_row * F]`. + +**Global barrier** — all tiles synchronize before validation. + +--- + +### Validation + +Tile 0 compares each element of `o_out` against the precomputed `o_golden` array. +Values are compared as integer millis (via `fp16_to_millis`) with an absolute threshold of +`0.008`. Errors and the final count are printed via `printf`. diff --git a/tests/magia/mesh/gemm_comm/via_L2/interlaced/src/test.c b/tests/magia/mesh/gemm_comm/via_L2/interlaced/src/test.c new file mode 100644 index 00000000..a92f3417 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L2/interlaced/src/test.c @@ -0,0 +1,413 @@ +// Copyright 2025 University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include "test.h" + +#include "tile.h" +#include "gemm_utils.h" +#include "fsync.h" +#include "idma.h" +#include "redmule.h" +#include "eventunit.h" + +#define WAIT_MODE WFE + +/** + * Tile group definitions for 8x8 mesh (64 tiles). + * + * GEMM1 tiles: [0, 1, 8, 9] (4 tiles, top-left 2x2 block) + * GEMM2 tiles: [16-19, 24-27, 32-35, + * 40-43, 48-51, 56-59] (24 tiles, rows 2-7 cols 0-3) + * GEMM3 tiles: [2-7, 10-15] (12 tiles, rows 0-1 cols 2-7) + * GEMM4 tiles: [20-23, 28-31, 36-39, + * 44-47, 52-55, 60-63] (24 tiles, rows 2-7 cols 4-7) + */ +#define GEMM1_N_TILES 4 +#define GEMM2_N_TILES 24 +#define GEMM3_N_TILES 12 +#define GEMM4_N_TILES 24 + +#define abs_threshold_millis 8 /* 0.008 expressed as integer millis */ + +static const uint32_t gemm1_tiles[GEMM1_N_TILES] = {0, 1, 8, 9}; + +static const uint32_t gemm2_tiles[GEMM2_N_TILES] = {16, 17, 18, 19, 24, 25, 26, 27, 32, 33, 34, 35, + 40, 41, 42, 43, 48, 49, 50, 51, 56, 57, 58, 59}; + +static const uint32_t gemm3_tiles[GEMM3_N_TILES] = {2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15}; + +static const uint32_t gemm4_tiles[GEMM4_N_TILES] = {20, 21, 22, 23, 28, 29, 30, 31, 36, 37, 38, 39, + 44, 45, 46, 47, 52, 53, 54, 55, 60, 61, 62, 63}; + +int get_local_idx(uint32_t hartid, const uint32_t *tiles, uint32_t n_tiles) +{ + for (uint32_t i = 0; i < n_tiles; i++) + if (tiles[i] == hartid) + return (int)i; + return -1; +} + +void get_row_range(uint32_t local_idx, + uint32_t n_tiles, + uint32_t total_rows, + uint32_t *start_row, + uint32_t *num_rows) +{ + uint32_t base = total_rows / n_tiles; + uint32_t rem = total_rows % n_tiles; + *start_row = local_idx * base + (local_idx < rem ? local_idx : rem); + *num_rows = base + (local_idx < rem ? 1 : 0); +} + +void mem_set_zero(uint32_t o, uint32_t dim) +{ + for (uint32_t i = 0; i < dim; i++) + mmio16(o + i * 2) = 0x0000; +} + +/** + * GEMM chain test with interlaced data movement. + * + * Compared to the naive L2 variant, GEMM4 tiles prefetch M5 during Phase 1 + * (overlapping with GEMM1/GEMM2 computation), and Phase 3 skips the M5 DMA. + * Phase 1 uses a local barrier instead of a global one. + * + * Phase 1 (parallel): + * GEMM1 group (4 tiles): R1[AxC] = M1[AxB] @ M2[BxC] + * GEMM2 group (24 tiles): R2[CxE] = M3[CxD] @ M4[DxE] + * GEMM4 group (24 tiles): prefetch M5[ExF] into L1 + * + * Phase 2: + * GEMM3 group (12 tiles): R3[AxE] = R1[AxC] @ R2[CxE] + * + * Phase 3: + * GEMM4 group (24 tiles): O[AxF] = R3[AxE] @ M5[ExF] (M5 already in L1) + * + * Each group splits output rows across its tiles. Tiles with no rows idle. + * Requires 8x8 mesh (64 tiles). + */ +int main(void) +{ + /** + * 0. Initializations + */ + uint32_t hartid = get_hartid(); + uint32_t x_id = GET_X_ID(hartid); + uint32_t y_id = GET_Y_ID(hartid); + uint32_t l1_tile_base = get_l1_base(hartid); + + // Init iDMA + idma_config_t idma_cfg = {.hartid = hartid}; + idma_controller_t idma_ctrl = { + .base = NULL, + .cfg = &idma_cfg, + .api = &idma_api, + }; + idma_init(&idma_ctrl); + + // Init RedMulE + redmule_config_t redmule_cfg = {.hartid = hartid}; + redmule_controller_t redmule_ctrl = { + .base = NULL, + .cfg = &redmule_cfg, + .api = &redmule_api, + }; + redmule_init(&redmule_ctrl); + + // Init FractalSync + fsync_config_t fsync_cfg = {.hartid = hartid}; + fsync_controller_t fsync_ctrl = { + .base = NULL, + .cfg = &fsync_cfg, + .api = &fsync_api, + }; + fsync_init(&fsync_ctrl); + +// Init the Event Unit controller +#if STALLING == 0 + eu_config_t eu_cfg = {.hartid = hartid}; + eu_controller_t eu_ctrl = { + .base = NULL, + .cfg = &eu_cfg, + .api = &eu_api, + }; + + eu_init(&eu_ctrl); + eu_clear_events(0xFFFFFFFF); + eu_fsync_init(&eu_ctrl, 0); + eu_idma_init(&eu_ctrl, 0); + eu_redmule_init(&eu_ctrl, 0); +#endif + + // Determine each tile's group membership up front (needed for interlaced scheduling) + int gemm1_idx = get_local_idx(hartid, gemm1_tiles, GEMM1_N_TILES); + int gemm2_idx = get_local_idx(hartid, gemm2_tiles, GEMM2_N_TILES); + int gemm3_idx = get_local_idx(hartid, gemm3_tiles, GEMM3_N_TILES); + int gemm4_idx = get_local_idx(hartid, gemm4_tiles, GEMM4_N_TILES); + + // Global barrier: ensure all tiles have completed startup (including BSS zeroing + // in crt0.S) before any tile begins writing results to L2 output buffers. + // Without this, slow tiles still zeroing BSS can overwrite Phase 1 results. + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + perf_reset(); + perf_start(); + + /** + * Phase 1: GEMM1 and GEMM2 in parallel, GEMM4 prefetches M5 + * GEMM1 group: R1 = M1 @ M2 + * GEMM2 group: R2 = M3 @ M4 + * GEMM4 group: DMA M5 into L1 at Phase 3 offset + */ + if (gemm1_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm1_idx, GEMM1_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: M1_slice, M2, R1_slice + uint32_t obi_m1 = l1_tile_base; + uint32_t obi_m2 = obi_m1 + (num_rows * DIM_B * 2); + uint32_t obi_r1 = obi_m2 + (DIM_B * DIM_C * 2); + + // Load slice of M1 [num_rows x B] from L2 + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m1_inp + start_row * DIM_B * 2, + obi_m1, + num_rows * DIM_B * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Load full M2 [BxC] from L2 + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m2_inp, obi_m2, DIM_B * DIM_C * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Zero accumulator and compute: R1_slice = M1_slice @ M2 + mem_set_zero(obi_r1, num_rows * DIM_C); + redmule_gemm(&redmule_ctrl, + obi_m1, + obi_m2, + obi_r1, + (uint16_t)num_rows, + (uint16_t)DIM_B, + (uint16_t)DIM_C); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Write R1 slice back to L2 at correct offset + idma_memcpy_1d(&idma_ctrl, + 1, + (uint32_t)r1_out + start_row * DIM_C * 2, + obi_r1, + num_rows * DIM_C * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + + if (gemm2_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm2_idx, GEMM2_N_TILES, DIM_C, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: M3_slice, M4, R2_slice + uint32_t obi_m3 = l1_tile_base; + uint32_t obi_m4 = obi_m3 + (num_rows * DIM_D * 2); + uint32_t obi_r2 = obi_m4 + (DIM_D * DIM_E * 2); + + // Load slice of M3 [num_rows x D] from L2 + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)m3_inp + start_row * DIM_D * 2, + obi_m3, + num_rows * DIM_D * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Load full M4 [DxE] from L2 + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m4_inp, obi_m4, DIM_D * DIM_E * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Zero accumulator and compute: R2_slice = M3_slice @ M4 + mem_set_zero(obi_r2, num_rows * DIM_E); + redmule_gemm(&redmule_ctrl, + obi_m3, + obi_m4, + obi_r2, + (uint16_t)num_rows, + (uint16_t)DIM_D, + (uint16_t)DIM_E); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Write R2 slice back to L2 at correct offset + idma_memcpy_1d(&idma_ctrl, + 1, + (uint32_t)r2_out + start_row * DIM_E * 2, + obi_r2, + num_rows * DIM_E * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + + if (gemm4_idx >= 0) { + // Prefetch M5 into L1 during Phase 1, placed at Phase 3 offset. + // Phase 3 L1 layout: R3_slice | M5 | O_slice + uint32_t start_row, num_rows; + get_row_range(gemm4_idx, GEMM4_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + uint32_t obi_m5 = l1_tile_base + (num_rows * DIM_E * 2); + + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)m5_inp, obi_m5, DIM_E * DIM_F * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + } + } + + // Local barrier: wait for Phase 1 to complete before GEMM3 reads r1_out / r2_out. + // TODO: use a lower sync level once a safe sub-global level is determined + // for the current tile topology. + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + /** + * Phase 2: GEMM3 (row-parallel) + * GEMM3 group: R3 = R1 @ R2 + */ + if (gemm3_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm3_idx, GEMM3_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: R1_slice, R2, R3_slice + uint32_t obi_r1 = l1_tile_base; + uint32_t obi_r2 = obi_r1 + (num_rows * DIM_C * 2); + uint32_t obi_r3 = obi_r2 + (DIM_C * DIM_E * 2); + + // Load slice of R1 [num_rows x C] from L2 + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)r1_out + start_row * DIM_C * 2, + obi_r1, + num_rows * DIM_C * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Load full R2 [CxE] from L2 + idma_memcpy_1d(&idma_ctrl, 0, (uint32_t)r2_out, obi_r2, DIM_C * DIM_E * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // Zero accumulator and compute: R3_slice = R1_slice @ R2 + mem_set_zero(obi_r3, num_rows * DIM_E); + redmule_gemm(&redmule_ctrl, + obi_r1, + obi_r2, + obi_r3, + (uint16_t)num_rows, + (uint16_t)DIM_C, + (uint16_t)DIM_E); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Write R3 slice back to L2 at correct offset + idma_memcpy_1d(&idma_ctrl, + 1, + (uint32_t)r3_out + start_row * DIM_E * 2, + obi_r3, + num_rows * DIM_E * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + + // Global barrier: wait for Phase 2 to complete + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + /** + * Phase 3: GEMM4 (row-parallel) — M5 already in L1 from Phase 1 prefetch + * GEMM4 group: O = R3 @ M5 + */ + if (gemm4_idx >= 0) { + uint32_t start_row, num_rows; + get_row_range(gemm4_idx, GEMM4_N_TILES, DIM_A, &start_row, &num_rows); + + if (num_rows > 0) { + // L1 layout: R3_slice | M5 (prefetched in Phase 1) | O_slice + uint32_t obi_r3 = l1_tile_base; + uint32_t obi_m5 = obi_r3 + (num_rows * DIM_E * 2); + uint32_t obi_o = obi_m5 + (DIM_E * DIM_F * 2); + + // Load slice of R3 [num_rows x E] from L2 + idma_memcpy_1d(&idma_ctrl, + 0, + (uint32_t)r3_out + start_row * DIM_E * 2, + obi_r3, + num_rows * DIM_E * 2); + eu_idma_wait_a2o(&eu_ctrl, WAIT_MODE); + + // M5 already in L1 at obi_m5 — no DMA needed + + // Zero accumulator and compute: O_slice = R3_slice @ M5 + mem_set_zero(obi_o, num_rows * DIM_F); + redmule_gemm(&redmule_ctrl, + obi_r3, + obi_m5, + obi_o, + (uint16_t)num_rows, + (uint16_t)DIM_E, + (uint16_t)DIM_F); + eu_redmule_wait(&eu_ctrl, WAIT_MODE); + + // Write O slice back to L2 at correct offset + idma_memcpy_1d(&idma_ctrl, + 1, + (uint32_t)o_out + start_row * DIM_F * 2, + obi_o, + num_rows * DIM_F * 2); + eu_idma_wait_o2a(&eu_ctrl, WAIT_MODE); + } + } + + // Global barrier: wait for Phase 3 to complete + fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); + eu_fsync_wait(&eu_ctrl, WAIT_MODE); + + perf_stop(); + + /** + * Validation: Tile 0 checks O against golden + */ + uint32_t errors = 0; + + if (hartid == 0) { + for (uint32_t i = 0; i < DIM_A; i++) { + for (uint32_t j = 0; j < DIM_F; j++) { + float16 computed = *(volatile float16 *)(&o_out[i * DIM_F + j]); + float16 expected = o_golden[i * DIM_F + j]; + + uint16_t uc = *(uint16_t *)&computed; + uint16_t ue = *(uint16_t *)&expected; + + int32_t vc = fp16_to_millis(uc); + int32_t ve = fp16_to_millis(ue); + + int32_t abs_diff = vc - ve; + if (abs_diff < 0) + abs_diff = -abs_diff; + if (abs_diff > abs_threshold_millis) { + errors++; + + printf("O[%d][%d]: got=%f (0x%x) exp=%f (0x%x) (abs_diff=%ld)\n", + i, + j, + fp16_to_f64(uc), + uc, + fp16_to_f64(ue), + ue, + (long)abs_diff); + } + } + } + + printf("\nTest complete. Errors: %d / %d\n\n", errors, DIM_A * DIM_F); + printf("Cycles: %u\n", perf_get_cycles()); + } + + return errors; +} diff --git a/tests/magia/mesh/gemm/via_l2/CMakeLists.txt b/tests/magia/mesh/gemm_comm/via_L2/naive/CMakeLists.txt similarity index 77% rename from tests/magia/mesh/gemm/via_l2/CMakeLists.txt rename to tests/magia/mesh/gemm_comm/via_L2/naive/CMakeLists.txt index fb90c830..adab9ecb 100644 --- a/tests/magia/mesh/gemm/via_l2/CMakeLists.txt +++ b/tests/magia/mesh/gemm_comm/via_L2/naive/CMakeLists.txt @@ -1,7 +1,7 @@ -set(TEST_NAME test_gemm) +set(TEST_NAME test_gemm_l2_naive) set(GEMM_TEST_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/include/test.h) -set(GEN_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/gen_golden.py) +set(GEN_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/../../gen_golden.py) add_custom_command( OUTPUT ${GEMM_TEST_HEADER} @@ -13,7 +13,7 @@ add_custom_command( file(GLOB_RECURSE TEST_SRCS "src/*.c") add_executable(${TEST_NAME} ${TEST_SRCS} ${GEMM_TEST_HEADER}) -target_include_directories(${TEST_NAME} PUBLIC include) +target_include_directories(${TEST_NAME} PUBLIC include ${CMAKE_CURRENT_SOURCE_DIR}/../..) target_compile_options(${TEST_NAME} PRIVATE -O2) target_link_libraries(${TEST_NAME} PUBLIC runtime hal) diff --git a/tests/magia/mesh/gemm_comm/via_L2/naive/README.md b/tests/magia/mesh/gemm_comm/via_L2/naive/README.md new file mode 100644 index 00000000..1323a250 --- /dev/null +++ b/tests/magia/mesh/gemm_comm/via_L2/naive/README.md @@ -0,0 +1,105 @@ +# via_l2_naive — GEMM Chain Test (L2 Communication) + +4-GEMM chain with task-level and row-parallel data parallelism across a 64-tile (8×8) mesh. +Inter-tile communication happens entirely through shared L2 memory: tiles DMA intermediate results +into L2 buffers that subsequent phases read back. + +## Overview + +- Phase 1: + - Move data L2->L1 (GEMM 1 and GEMM 2) + - GEMM 1 and GEMM 2 + - Move result L1->L2 (GEMM 1 and GEMM 2) + - === Global barrier === + +- Phase 2: + - Move data L2->L1 (GEMM 3) + - GEMM 3 + - Move result L1->L2 (GEMM 3) + - === Global barrier === + +- Phase 3: + - Move data L2->L1 (GEMM 4) + - GEMM 4 + - Move result L1->L2 (GEMM 4) + - === Global barrier === + +## Tile Groups + +| Group | Count | Role | +|---------|-------|---------------| +| GEMM 1 | 4 | Phase 1 | +| GEMM 2 | 24 | Phase 1 | +| GEMM 3 | 12 | Phase 2 | +| GEMM 4 | 24 | Phase 3 | + +## Phases + +### Phase 0 — Initialization +1. Each tile reads its `hartid`, derives `x_id`, `y_id`, and its L1 base address. +2. iDMA, RedMulE, FractalSync, and EventUnit controllers are initialized. +3. **Global barrier** (FractalSync at `MAX_SYNC_LVL - 1`): all tiles wait until every tile has + finished BSS zeroing (performed by tile 0 in `crt0.S`) before any tile writes to L2 output + buffers. + +--- + +### Phase 1 — Parallel GEMMs (GEMM1 ∥ GEMM2) + +Both groups run concurrently. Within each group, output rows are split evenly across tiles. + +**GEMM1** — `R1[A×C] = M1[A×B] @ M2[B×C]` + +For each tile in the group: +1. Compute the tile's row slice (`start_row`, `num_rows`) of the output. +2. DMA `M1[num_rows × B]` from L2 into L1. +3. DMA full `M2[B × C]` from L2 into L1. +4. Zero the L1 accumulator buffer for `R1_slice`. +5. RedMulE GEMM: `R1_slice = M1_slice @ M2`. +6. DMA `R1_slice` back to L2 at `r1_out[start_row * C]`. + +**GEMM2** — `R2[C×E] = M3[C×D] @ M4[D×E]` + +Same steps as GEMM1, operating on `M3`, `M4` → `r2_out` (`R2_slice = M3_slice @ M4`). + +**Global barrier** — all tiles synchronize before Phase 2 reads `r1_out` / `r2_out`. + +--- + +### Phase 2 — GEMM3 + +**GEMM3** — `R3[A×E] = R1[A×C] @ R2[C×E]` + +For each tile in the group: +1. Compute the tile's row slice of `R3`. +2. DMA `R1[num_rows × C]` from L2 (`r1_out`) into L1. +3. DMA full `R2[C × E]` from L2 (`r2_out`) into L1. +4. Zero the L1 accumulator buffer for `R3_slice`. +5. RedMulE GEMM: `R3_slice = R1_slice @ R2`. +6. DMA `R3_slice` back to L2 at `r3_out[start_row * E]`. + +**Global barrier** — all tiles synchronize before Phase 3 reads `r3_out`. + +--- + +### Phase 3 — GEMM4 + +**GEMM4** — `O[A×F] = R3[A×E] @ M5[E×F]` + +For each tile in the group: +1. Compute the tile's row slice of `O`. +2. DMA `R3[num_rows × E]` from L2 (`r3_out`) into L1. +3. DMA full `M5[E × F]` from L2 into L1. +4. Zero the L1 accumulator buffer for `O_slice`. +5. RedMulE GEMM: `O_slice = R3_slice @ M5`. +6. DMA `O_slice` back to L2 at `o_out[start_row * F]`. + +**Global barrier** — all tiles synchronize before validation. + +--- + +### Validation + +Tile 0 compares each element of `o_out` against the precomputed `o_golden` array. +Values are compared as integer millis (via `fp16_to_millis`) with an absolute threshold of +`0.008`. Errors and the final count are printed via `printf`. diff --git a/tests/magia/mesh/gemm/via_l2/src/test.c b/tests/magia/mesh/gemm_comm/via_L2/naive/src/test.c similarity index 99% rename from tests/magia/mesh/gemm/via_l2/src/test.c rename to tests/magia/mesh/gemm_comm/via_L2/naive/src/test.c index fef6b137..ec21d956 100644 --- a/tests/magia/mesh/gemm/via_l2/src/test.c +++ b/tests/magia/mesh/gemm_comm/via_L2/naive/src/test.c @@ -6,7 +6,7 @@ #include "test.h" #include "tile.h" -#include "utils/gemm_utils.h" +#include "gemm_utils.h" #include "fsync.h" #include "idma.h" #include "redmule.h" @@ -142,6 +142,9 @@ int main(void) fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); eu_fsync_wait(&eu_ctrl, WAIT_MODE); + perf_reset(); + perf_start(); + /** * Phase 1: GEMM1 and GEMM2 in parallel (row-parallel within each group) * GEMM1 group: R1 = M1 @ M2 @@ -343,6 +346,8 @@ int main(void) fsync_sync_level(&fsync_ctrl, MAX_SYNC_LVL - 1, 0); eu_fsync_wait(&eu_ctrl, WAIT_MODE); + perf_stop(); + /** * Validation: Tile 0 checks O against golden */ @@ -379,6 +384,7 @@ int main(void) } printf("\nTest complete. Errors: %d / %d\n\n", errors, DIM_A * DIM_F); + printf("Cycles: %u\n", perf_get_cycles()); } return errors;