ci.yml

# CI workflow - builds, tests, and scans for vulnerabilities
# Runs on pushes to main and all pull requests

name: CI

on:
  push:
    branches: [ "main" ]
  pull_request:  # This will run on ALL pull requests regardless of source/target branch

permissions:
  contents: read

# Within one run the jobs fan out across every available runner. Across runs, push
# builds on main are serialised: ldaps-tests mutates machine-global state on the
# self-hosted host (fixed container names, /etc/hosts entries, a trust-store CA),
# so two overlapping main runs would corrupt each other's fixtures. A newer queued
# main run supersedes an older queued one, which is fine: the newest commit contains
# the older ones. PR runs get one group per PR so a new push cancels the stale run.
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'push' && 'main-push' || github.ref }}
  cancel-in-progress: ${{ github.event_name != 'push' }}

jobs:
  build-and-test:

    # Split-trust runner model: push events only happen once code is on main (merged by
    # an authorised person), so they may use the fast self-hosted runner. pull_request
    # events execute code from anyone's fork and must stay on GitHub-hosted VMs; a
    # self-hosted runner on a public repo would hand fork authors code execution on our
    # infrastructure. The org runner group must also allowlist this workflow at
    # refs/heads/main only, so that a fork PR editing this expression still cannot
    # reach the runner. Every job in this file uses the same expression.
    runs-on: ${{ github.event_name == 'push' && 'self-hosted' || 'ubuntu-latest' }}

    steps:
    - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
    - name: Setup .NET
      uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9  # v4.3.1
      with:
        dotnet-version: 10.0.x
    - name: Restore dependencies
      run: dotnet restore JIM.sln --locked-mode
    - name: Build
      run: dotnet build JIM.sln --no-restore
    - name: Test .NET
      run: dotnet test JIM.sln --no-build --verbosity normal

    # Every source file must carry the Tetron copyright notice. .editorconfig
    # only ever covered .cs (and only as a suggestion), so this is the gate for
    # .razor, .ps1, .psm1, .psd1 and .sh.
    - name: Lint copyright headers
      shell: pwsh
      run: pwsh -File ./scripts/Lint-CopyrightHeaders.ps1

    # PowerShell Pester Tests for the JIM PowerShell module
    - name: Install Pester
      shell: pwsh
      run: |
        Set-PSRepository PSGallery -InstallationPolicy Trusted
        Install-Module -Name Pester -MinimumVersion 5.0 -Force -Scope CurrentUser
    - name: Run Pester Tests
      shell: pwsh
      run: |
        $config = New-PesterConfiguration
        $config.Run.Path = @("./src/JIM.PowerShell/Tests", "./scripts/Tests", "./test/integration/utils", "./.github/scripts/Tests")
        $config.Run.Exit = $true
        $config.Output.Verbosity = "Detailed"
        $config.TestResult.Enabled = $true
        $config.TestResult.OutputPath = "./TestResults/pester-results.xml"
        $config.TestResult.OutputFormat = "NUnitXml"
        Invoke-Pester -Configuration $config
    - name: Upload Pester Test Results
      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
      if: always()
      with:
        name: pester-test-results
        path: ./TestResults/pester-results.xml

  openapi-document:
    # Generate the OpenAPI document, and fail if it cannot be generated.
    #
    # The jim.web image bakes the document in at build time (the openapi-gen stage in
    # src/JIM.Web/Dockerfile), so a document that cannot be generated is an image that
    # cannot be built, and a release that cannot be cut. Generating it boots the app far
    # enough to walk every route and every response type, which is why it catches a class
    # of fault nothing else here can: duplicate route parameters, ambiguous templates,
    # and reference cycles in the model graph. dotnet build stays clean through all of
    # them, and unit tests call action methods directly, so they never touch routing or
    # schema generation either.
    #
    # It is a required check because the alternative was demonstrated: #1238 shipped a
    # cycle between ConnectedSystemObjectType and its tags, every check passed, and no
    # release image could be built until it was found by hand days later. Local builds do
    # not cover it either, deliberately: jim-build passes OPENAPI_STAGE=publish to skip
    # the stage because it costs minutes.
    #
    # Its own job rather than a step in build-and-test: generation alone takes about five
    # minutes (measured in both Debug and Release; it is the schema walk over the model
    # graph, not the build), and it needs nothing the tests produce. Run alongside them
    # it costs no wall clock; appended to them it would add five minutes to every PR.
    runs-on: ${{ github.event_name == 'push' && 'self-hosted' || 'ubuntu-latest' }}

    steps:
    - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
    - name: Setup .NET
      uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9  # v4.3.1
      with:
        dotnet-version: 10.0.x
    - name: Restore dependencies
      run: dotnet restore JIM.sln --locked-mode
    - name: Build
      run: dotnet build JIM.sln --no-restore
    # The same script the devcontainer uses (jim-openapi-generate), so what CI proves and
    # what a developer can reproduce are the same code path. -NoBuild reuses the build
    # above. The output goes to the runner's temp directory: the document is a build
    # artefact of the image, not a file this repository tracks.
    - name: Generate the OpenAPI document
      shell: pwsh
      run: ./scripts/Generate-OpenApiDoc.ps1 -NoBuild -OutputPath "${{ runner.temp }}/openapi-v1.json"

  database-tests:
    # Runs the RequiresPostgres NUnit tier (the "Database-backed component tests"
    # tier in engineering/TESTING_STRATEGY.md) against a real PostgreSQL service.
    # These fixtures self-skip in the in-memory build-and-test job (JIM_TEST_RESET_DB
    # unset); here they execute, guarding provider-specific and raw-SQL behaviour the
    # EF Core in-memory provider cannot reproduce (the class of bug behind #849/#850).
    # See issue #861. Runs on every PR so a failure blocks the merge.
    runs-on: ${{ github.event_name == 'push' && 'self-hosted' || 'ubuntu-latest' }}

    services:
      postgres:
        # Pinned to match the production database image in docker-compose.yml; bump
        # both together. Dependabot's docker ecosystem does not scan workflow service
        # images, so this digest pin is maintained by hand.
        image: postgres:18.4@sha256:4aabea78cf39b90e834caf3af7d602a18565f6fe2508705c8d01aa63245c2e20
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: jim_test
        # Ephemeral host port. A fixed 5432:5432 collides on the self-hosted runner
        # host, where a second runner service can be running another job (e.g. a
        # JIM-Bench run) with its own PostgreSQL on the same port at the same moment.
        ports:
          - 5432
        # Hold the job's steps until PostgreSQL is accepting connections.
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    # The RequiresPostgres fixtures read these to build the connection string and to
    # opt in (JIM_TEST_RESET_DB being set is what flips them from Assert.Ignore to run).
    # jim_test is a throwaway database: each fixture migrates the schema once and
    # TRUNCATEs every table between tests.
    env:
      JIM_TEST_RESET_HOST: localhost
      JIM_TEST_RESET_USER: postgres
      JIM_TEST_RESET_PASSWORD: postgres
      JIM_TEST_RESET_DB: jim_test

    steps:
    - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
    - name: Setup .NET
      uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9  # v4.3.1
      with:
        dotnet-version: 10.0.x
    - name: Restore dependencies
      run: dotnet restore JIM.sln --locked-mode
    - name: Build
      run: dotnet build JIM.sln --no-restore
    - name: Run database-backed tests (RequiresPostgres)
      # The ephemeral host port must be read here rather than in the job-level env block
      # above: the job context is only available inside steps, and referencing it at job
      # level invalidates the whole workflow file (GitHub reports "workflow file issue"
      # and no CI jobs run at all).
      env:
        JIM_TEST_RESET_PORT: ${{ job.services.postgres.ports['5432'] }}
      run: dotnet test JIM.sln --no-build --filter "Category=RequiresPostgres" --verbosity normal

  ldaps-tests:
    # Runs the RequiresLdaps NUnit tier against real directory servers over TLS:
    # three OpenLDAP variants (system-trusted CA, JIM-store-only CA, expired
    # certificate) and a Samba AD Domain Controller. These fixtures self-skip in
    # build-and-test (JIM_TEST_LDAPS_HOST unset); here the fixture script stands
    # the servers up and exports the environment variables that flip them from
    # Assert.Ignore to run. This is the regression net for LDAPS certificate
    # validation (#1132): connections must be refused for untrusted issuers, name
    # mismatches and expired certificates, with the reason reported, and must
    # succeed when the CA is in the JIM certificate store. See issue #1141.
    # Runs on every PR so a failure blocks the merge.
    #
    # LDAPTLS_REQCERT must never be set here: it would disable libldap certificate
    # validation for the whole process and turn every refusal case into a false pass,
    # which is the exact defect class this job exists to catch. A guard step asserts it.
    runs-on: ${{ github.event_name == 'push' && 'self-hosted' || 'ubuntu-latest' }}

    steps:
    - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
    - name: Setup .NET
      uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9  # v4.3.1
      with:
        dotnet-version: 10.0.x
    - name: Restore dependencies
      run: dotnet restore JIM.sln --locked-mode
    - name: Build
      run: dotnet build test/JIM.Worker.Tests/ --no-restore
    - name: Assert LDAPTLS_REQCERT is not in scope
      run: |
        if [ -n "${LDAPTLS_REQCERT}" ]; then
          echo "LDAPTLS_REQCERT is set; it would disable certificate validation and void this job."
          exit 1
        fi
    # Docker Hub rate-limits anonymous pulls (429s observed in testing), so pull the
    # fixture images up front with a retry rather than letting the fixture script's
    # docker run fail on a transient refusal.
    - name: Pull directory server images
      run: |
        for image in bitnamilegacy/openldap:latest diegogslomp/samba-ad-dc:latest; do
          for attempt in 1 2 3; do
            docker pull "$image" && break
            if [ "$attempt" = "3" ]; then
              echo "Failed to pull $image after 3 attempts."
              exit 1
            fi
            sleep 30
          done
        done
    # The script needs root: it writes /etc/hosts entries for the certificate names
    # and installs one test CA into the machine trust store. sudo -E preserves
    # GITHUB_ENV so the script can export the connection details to later steps.
    - name: Start LDAPS certificate test servers
      run: sudo -E pwsh -File ./test/scripts/Start-LdapsCertificateTestServers.ps1 -IncludeSambaAd
    - name: Run LDAPS certificate validation tests (RequiresLdaps)
      run: dotnet test test/JIM.Worker.Tests/ --no-build --filter "Category=RequiresLdaps" --verbosity normal
    # On push events this job runs on the self-hosted runner, which is NOT ephemeral:
    # without teardown the containers, /etc/hosts entries and the trust-store CA
    # persist on the machine between jobs. Harmless on GitHub-hosted PR runs, essential
    # on the self-hosted host; run it unconditionally.
    - name: Tear down LDAPS certificate test servers
      if: always()
      run: sudo pwsh -File ./test/scripts/Start-LdapsCertificateTestServers.ps1 -Stop

  discover-base-images:
    # Discover production Dockerfiles and enforce the digest-pinning policy.
    # Production Dockerfiles are identified by the "# jim-compliance: production-image"
    # directive; unlabelled Dockerfiles (devcontainer, test fixtures) are out of scope.
    # Any production Dockerfile with a non-digest-pinned FROM line fails here. The job
    # emits two matrices: the unique base image references (the pin policy's subject)
    # and the production Dockerfiles themselves, which scan-images builds and scans.
    # See .github/scripts/discover-base-images.ps1 and engineering/DEVELOPER_GUIDE.md.
    runs-on: ${{ github.event_name == 'push' && 'self-hosted' || 'ubuntu-latest' }}
    outputs:
      matrix: ${{ steps.discover.outputs.matrix }}
      image_matrix: ${{ steps.discover.outputs.image_matrix }}

    steps:
    - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0

    - name: Discover production base images and enforce digest-pinning
      id: discover
      shell: pwsh
      run: ./.github/scripts/discover-base-images.ps1

  scan-images:
    # Build each production JIM image and scan it for CRITICAL/HIGH CVEs with a
    # published fix. Emits SARIF so findings are surfaced in the GitHub Security tab,
    # not just the Actions log. Catches vulnerabilities early, before a release tag
    # is created.
    #
    # The built image, not the base image, is the subject: it is what customers run,
    # and it alone carries the apt pins and the build-time apt-get upgrade (see any
    # production Dockerfile). Scanning the base image instead reported every Ubuntu
    # advisory Microsoft's rebuild had not yet caught up with, none of which the
    # shipped image still contained, and could not see the JIM layer at all. The
    # matrix comes from discover-base-images, so adding a new production Dockerfile
    # requires zero changes to this workflow: just add the
    # "# jim-compliance: production-image" directive to the new file.
    needs: discover-base-images
    runs-on: ${{ github.event_name == 'push' && 'self-hosted' || 'ubuntu-latest' }}
    permissions:
      contents: read
      security-events: write  # required to upload SARIF to code scanning

    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.discover-base-images.outputs.image_matrix) }}

    steps:
    - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f  # v3.12.0

    # Same action, pins and cache scope as the release build, so a PR's build warms the
    # cache the release will use and vice versa. The image is loaded into the local
    # daemon for Trivy to read via --image-src docker; nothing is pushed. OPENAPI_STAGE
    # skips the JIM.Web OpenAPI generation stage, which costs about five minutes and
    # contributes nothing to what is scanned; the openapi-document job proves it separately.
    - name: Build image
      uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8  # v6.19.2
      with:
        context: .
        file: ${{ matrix.dockerfile }}
        load: true
        tags: ${{ matrix.image_name }}:scan
        build-args: |
          VERSION=scan
          OPENAPI_STAGE=publish
        cache-from: type=gha,scope=${{ matrix.image_name }}
        cache-to: type=gha,scope=${{ matrix.image_name }},mode=max

    - name: Scan image for vulnerabilities
      id: trivy
      # Run Trivy from its pinned container image rather than via
      # aquasecurity/trivy-action + setup-trivy. The action path re-downloaded
      # the Trivy binary from the GitHub releases API on every run, which failed
      # intermittently on GitHub-hosted runners ("unable to find 'vX.Y.Z'"), and
      # the action wrapper independently exited 1 without writing a SARIF
      # (phantom failure). Both flakes blocked unrelated PRs at random. Calling
      # the pinned binary directly removes the release-API lookup entirely and
      # lets us own the exit code. See PR #520 and #800.
      #
      # github.token authenticates the vuln-DB pull from ghcr.io, avoiding the
      # anonymous rate limit (the one external dependency left on the hot path).
      #
      # No severity filter on the scan: we emit every finding to the SARIF and
      # apply our own CVSS-based gate in the next step (more reliable than the
      # action's filter, and uses the same score GitHub Code Scanning shows).
      # ignore-unfixed stays on: no value blocking on CVEs with no upstream fix.
      #
      # The image tag is passed via env, not interpolated into the run script, to
      # avoid GitHub Actions expression-injection (flagged by CodeQL actions).
      env:
        # aquasec/trivy:0.70.0 pinned by digest (supply-chain governance).
        TRIVY_IMAGE: 'aquasec/trivy:0.70.0@sha256:be1190afcb28352bfddc4ddeb71470835d16462af68d310f9f4bca710961a41e'
        IMAGE_REF: ${{ matrix.image_name }}:scan
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        docker run --rm \
          -v /var/run/docker.sock:/var/run/docker.sock \
          -v "$GITHUB_WORKSPACE:/workspace" \
          -w /workspace \
          -e GITHUB_TOKEN \
          -e TRIVY_IGNORE_UNFIXED=true \
          "$TRIVY_IMAGE" image \
          --scanners vuln \
          --format sarif \
          --output trivy-results.sarif \
          --ignorefile .trivyignore \
          --image-src docker \
          --exit-code 0 \
          "$IMAGE_REF"

    - name: Fail build on fixable HIGH/CRITICAL Trivy findings
      shell: pwsh
      run: |
        if (-not (Test-Path 'trivy-results.sarif')) {
          Write-Error "Trivy did not produce a SARIF file."
          exit 1
        }

        $sarif = Get-Content 'trivy-results.sarif' -Raw | ConvertFrom-Json

        # Build a ruleId -> CVSS score map from the SARIF rules catalogue.
        # Trivy writes the CVSS base score to rule.properties.'security-severity'
        # as a decimal string (e.g. "7.5"). This is the same field GitHub Code
        # Scanning reads to classify alerts as low/medium/high/critical.
        $severityByRule = @{}
        foreach ($r in $sarif.runs) {
          foreach ($rule in $r.tool.driver.rules) {
            $score = 0.0
            $raw = $rule.properties.'security-severity'
            if ($raw) { [void][double]::TryParse($raw, [ref]$score) }
            $severityByRule[$rule.id] = $score
          }
        }

        # Walk results, classify by CVSS, and count only HIGH (>= 7.0) and
        # CRITICAL (>= 9.0). Also collect a sample for the log.
        $critical = 0
        $high     = 0
        $medium   = 0
        $low      = 0
        $blocking = @()
        foreach ($r in $sarif.runs) {
          foreach ($result in @($r.results)) {
            if (-not $result) { continue }
            $ruleId = $result.ruleId
            $score  = [double]$severityByRule[$ruleId]
            if ($score -ge 9.0) {
              $critical++
              $blocking += "$ruleId (CVSS $score)"
            } elseif ($score -ge 7.0) {
              $high++
              $blocking += "$ruleId (CVSS $score)"
            } elseif ($score -ge 4.0) {
              $medium++
            } else {
              $low++
            }
          }
        }

        Write-Host "Trivy findings for ${{ matrix.image_name }} (${{ matrix.dockerfile }}):"
        Write-Host "  CRITICAL: $critical"
        Write-Host "  HIGH:     $high"
        Write-Host "  MEDIUM:   $medium  (not blocking)"
        Write-Host "  LOW:      $low  (not blocking)"
        Write-Host ''

        $blockingCount = $critical + $high
        if ($blockingCount -gt 0) {
          Write-Host "Blocking CVEs (CVSS >= 7.0):"
          $blocking | Sort-Object -Unique | ForEach-Object { Write-Host "  $_" }
          Write-Host ''
          Write-Host 'Fixable CRITICAL/HIGH vulnerabilities found. Failing build.'
          Write-Host 'See the GitHub Security tab for details:'
          Write-Host '  https://github.com/TetronIO/JIM/security/code-scanning'
          exit 1
        }
        Write-Host 'No fixable CRITICAL/HIGH vulnerabilities found.'

    - name: Upload Trivy scan results to GitHub code scanning
      if: always() && hashFiles('trivy-results.sarif') != ''
      uses: github/codeql-action/upload-sarif@6f5948dfacef28e207b48d0905cf90c03365536d  # v3.37.9
      with:
        sarif_file: trivy-results.sarif
        # Category is keyed by the JIM image name (stable across base image
        # digest bumps and Dockerfile edits), so fixed findings close on the
        # next upload rather than sitting orphaned under a category that is
        # never uploaded again. See discover-base-images.ps1.
        category: trivy-image-${{ matrix.image_name }}

  scan-base-images-summary:
    # Stable-name gate for the dynamic scan-images matrix, providing the single
    # check name the branch protection ruleset requires. The name predates the
    # move from scanning base images to scanning the built JIM images and is
    # kept because the ruleset names it; renaming it here without changing the
    # ruleset would leave every PR waiting on a check that never reports.
    if: always()
    needs: [discover-base-images, scan-images]
    runs-on: ${{ github.event_name == 'push' && 'self-hosted' || 'ubuntu-latest' }}
    steps:
    - name: Check matrix results
      run: |
        if [ "${{ needs.discover-base-images.result }}" != "success" ]; then
          echo "discover-base-images failed or was cancelled"
          exit 1
        fi
        if [ "${{ needs.scan-images.result }}" != "success" ]; then
          echo "One or more scan-images matrix legs failed"
          exit 1
        fi
        echo "All image scans passed"