spack.yml

name: Spack build and run

# Build a version of the benchmark code using the Spack spec. Executes
# the CPU version of the benchmark.

on:
  # Uncomment the below 'push' to trigger on push
  # push:
  #   branches:
  #     - "**"
  # pull_request:
  #   branches: [ "main" ]
  # merge_group:
  #   branches:
  #     - main
  # schedule:
  #   # '*' is a special character in YAML, so string must be quoted
  #   - cron: "0 2 * * THU"
  workflow_dispatch:

env:
  GITHUB_USER: ${{ github.actor }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
  build:
    name: Build benchmark
    runs-on: ubuntu-24.04
    permissions:
      packages: write
    strategy:
      matrix:
        backend:
          [
            { device: cpu, spack_opt: "bench-dolfinx@main~cuda~rocm" },
            # Should be able to use "bench-dolfinx@main++rocm amdgpu_target==gfx90a ^openmpi"
            { device: rocm, spack_opt: "bench-dolfinx@main+rocm amdgpu_target=gfx90a ^openmpi+rocm amdgpu_target=gfx90a" },
            # Should be able to use "bench-dolfinx@main++cuda cuda_arch==75 ^openmpi"
            { device: cuda, spack_opt: "bench-dolfinx@main++cuda cuda_arch=75 ^openmpi" },
          ]

    steps:
      - uses: actions/checkout@v4

      - if: ${{ matrix.backend.device != 'cpu' }}
        name: Install GPU compiler/driver
        uses: ukri-bench/spack-configs/actions/install-gpu-driver@main
        with:
          runtime: ${{ matrix.backend.device }}

      - name: Set up Spack
        uses: spack/setup-spack@v2
        with:
          ref: develop     # Spack version (examples: develop, releases/v0.23)
          color: true      # Force color output (SPACK_COLOR=always)
          path: spack-src  # Where to clone Spack

      - if: ${{ matrix.backend.device != 'rocm' }}
        name: Get Spack config
        run: |
          wget -O spack.yml https://raw.githubusercontent.com/ukri-bench/spack-configs/refs/heads/main/configs/gh-actions/spack.yml
      - if: ${{ matrix.backend.device == 'rocm' }}
        name: Get Spack config
        run: |
          wget -O spack.yml https://raw.githubusercontent.com/ukri-bench/spack-configs/refs/heads/main/configs/gh-actions/spack-rocm.yml

      - name: Add ukri-bench Spack repository and create environment
        shell: spack-bash {0}
        run: |
          spack repo add --name bench_pkgs https://github.com/ukri-bench/spack-packages.git ./bench_pkgs
          spack mirror add --unsigned --type binary --oci-username-variable GITHUB_USER --oci-password-variable GITHUB_TOKEN local-buildcache oci://ghcr.io/ukri-bench/spack-buildcache
          spack env create . spack.yml

      - if: ${{ matrix.backend.device == 'rocm' }}
        name: Find compilers for ROCm
        shell: spack-bash {0}
        run: spack -e . compiler find
      - if: ${{ matrix.backend.device == 'cuda' }}
        name: Find CUDA
        shell: spack-bash {0}
        run: spack -e . external find -p /usr/local/cuda --all --not-buildable

      - name: Install benchmark
        shell: spack-bash {0}
        run: |
          spack -e . install -U -j 4 --use-buildcache=package:never,dependencies:auto \
            --add py-numpy ${{ matrix.backend.spack_opt }}

      - name: View Spack packages
        shell: spack-bash {0}
        run: spack -e . config get packages

      - name: Create mirror for runnable test image
        shell: spack-bash {0}
        run: |
          spack -e . mirror add --unsigned --type binary --oci-username-variable GITHUB_USER \
            --oci-password-variable GITHUB_TOKEN local-buildcache-${{ matrix.backend.device }} \
            oci://ghcr.io/ukri-bench/spack-buildcache-${{ matrix.backend.device }}

      - if: ${{ matrix.backend.device == 'cuda' }}
        name: Push runnable test image
        shell: spack-bash {0}
        run: |
          spack -e . buildcache push --base-image nvidia/cuda:12.9.1-base-ubuntu24.04 \
            --update-index --tag ${{ github.sha }} local-buildcache-${{ matrix.backend.device }}

      - if: ${{ matrix.backend.device != 'cuda' }}
        name: Push runnable test image
        shell: spack-bash {0}
        run: |
          spack -e . buildcache push --base-image ubuntu:24.04 \
            --update-index --tag ${{ github.sha }} local-buildcache-${{ matrix.backend.device }}

      - name: Push cache packages and update index
        shell: spack-bash {0}
        run: |
          spack -e . buildcache push --base-image ubuntu:24.04 --only dependencies \
            --with-build-dependencies --update-index local-buildcache
        if: ${{ !cancelled() }}

  run-code:
    name: Run benchmark (CPU)
    needs: build
    runs-on: ubuntu-latest
    container: ghcr.io/ukri-bench/spack-buildcache-cpu:${{ github.sha }}
    env:
      OMPI_ALLOW_RUN_AS_ROOT: 1
      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
      PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
    steps:
      - uses: actions/checkout@v4  # To get post-processing script
      - name: Run (CPU)
        run: |
          bench_dolfinx --ndofs=1000 --degree=3 --qmode=0 --nreps=1 --mat_comp --float=64 --json a.json
          mpirun -n 2 bench_dolfinx --ndofs=500 --degree=3 --qmode=0 --nreps=1 --mat_comp --float=64 --json b.json
      - name: Post-process
        run: |
          python src/test_output.py a.json
          python src/test_output.py b.json

  run-code-docker:
    name: Run benchmark in Docker (CPU)
    needs: build
    runs-on: ubuntu-24.04
    steps:
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Run (CPU)
        run: |
          docker run ghcr.io/ukri-bench/spack-buildcache-cpu:${{ github.sha }} \
            bench_dolfinx  --ndofs=10000 --degree=3 --qmode=0 --nreps=1 --mat_comp --float=64 --json a.json
          docker run --env OMPI_ALLOW_RUN_AS_ROOT=1 --env OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \
            ghcr.io/ukri-bench/spack-buildcache-cpu:e156f080fda032cb1d60ea0fdfd0aa82ac226e8c \
            mpirun -np 2 bench_dolfinx  --ndofs=10000 --degree=3 --qmode=0 --nreps=1 --mat_comp --float=64 --json a.json

  run-cuda-aws:
    name: Run benchmark (CUDA) on AWS
    needs: build
    uses: ./.github/workflows/runs-on-ci.yml
    with:
      tag: ${{ github.sha }}

  # run-code-aws:
  #   name: Run benchmark on AWS CodeBuild
  #   needs: build
  #   runs-on:
  #     - codebuild-ukri-bench-${{ github.run_id }}-${{ github.run_attempt }}
  #   container:
  #     image: ghcr.io/ukri-bench/spack-buildcache-cpu:${{ github.sha }}
  #     credentials:
  #       username: ${{ github.actor }}
  #       password: ${{ secrets.github_token }}
  #   env:
  #     OMPI_ALLOW_RUN_AS_ROOT: 1
  #     OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
  #     PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
  #   steps:
  #     - uses: actions/checkout@v4  # To get post-processing script
  #     - name: Run (CPU)
  #       run: |
  #         bench_dolfinx --ndofs=1000 --degree=3 --qmode=0 --nreps=1 --mat_comp --float=64 --json a.json
  #         mpirun -n 2 bench_dolfinx --ndofs=500 --degree=3 --qmode=0 --nreps=1 --mat_comp --float=64 --json b.json
  #     - name: Post-process
  #       run: |
  #         python src/test_output.py a.json
  #         python src/test_output.py b.json

  # run-code-azure:
  #   name: Trigger Azure Pipeline (run)
  #   needs: build
  #   runs-on: ubuntu-latest
  #   steps:
  #   # - uses: Azure/pipelines@v1.2
  #   - uses: tjcorr/pipelines@v1.4
  #     with:
  #       azure-devops-project-url: 'https://dev.azure.com/ukribench/benchmark-dolfinx'
  #       azure-pipeline-name: 'benchmark-dolfinx.cuda'  # name of the Azure pipeline to be triggered
  #       azure-devops-token: ${{ secrets.AZURE_DEVOPS_TOKEN }}
  #       azure-template-parameters: '{"IMAGE_NAME": "spack-buildcache-cuda", "TAG": "${{ github.sha }}"}'

  run-cpu-reusable:
    name: Run benchmark (CPU, reusable)
    needs: build
    uses: ./.github/workflows/run_cpu.yml
    with:
      tag: ${{ github.sha }}