action.yml

name: AuditAI
description: >
  Run LLM/RAG safety audits (faithfulness, relevancy, prompt injection)
  in CI and gate Pull Requests. Thin wrapper around the AuditAI CLI.
author: AuditAI

branding:
  icon: shield
  color: purple

inputs:
  config:
    description: Path to auditai.yml (relative to working-directory)
    required: false
    default: auditai.yml
  working-directory:
    description: Directory that contains the config and dataset
    required: false
    default: .
  fail-on:
    description: Override run.fail_on — average | any (empty = use YAML)
    required: false
    default: ""
  out:
    description: Output directory for reports
    required: false
    default: auditai-out
  baseline:
    description: >
      Optional path to a known-good auditai-report.json, relative to
      working-directory. When set, the Action also runs the regression gate.
    required: false
    default: ""
  max-drop:
    description: Maximum absolute metric-mean drop allowed against baseline
    required: false
    default: "0.05"
  python-version:
    description: Python version for the runner
    required: false
    default: "3.11"
  install:
    description: >
      Pip install target for AuditAI. Use "auditai-cli" for PyPI, or a path/URL
      (e.g. "." or "git+https://github.com/iZenDeveloper/auditai.git@v0.1.2").
    required: false
    default: auditai-cli
  comment-on-pr:
    description: Post Markdown report as a PR comment when running on pull_request
    required: false
    default: "true"
  upload-artifact:
    description: Upload auditai-out as a workflow artifact
    required: false
    default: "true"
  github-token:
    description: Token used for PR comments (defaults to GITHUB_TOKEN)
    required: false
    default: ${{ github.token }}

outputs:
  exit-code:
    description: CLI exit code (0 pass, 1 audit failed, 2 config, 3 internal)
    value: ${{ steps.audit.outputs.exit-code }}
  passed:
    description: Set to true when audit thresholds passed
    value: ${{ steps.audit.outputs.passed }}
  report-md:
    description: Path to Markdown report
    value: ${{ steps.audit.outputs.report-md }}
  report-json:
    description: Path to JSON report
    value: ${{ steps.audit.outputs.report-json }}

runs:
  using: composite
  steps:
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ inputs.python-version }}

    - name: Install AuditAI
      shell: bash
      working-directory: ${{ inputs.working-directory }}
      run: |
        set -euo pipefail
        python -m pip install --upgrade pip
        python -m pip install "${{ inputs.install }}"
        auditai --version

    - name: Run audit
      id: audit
      shell: bash
      working-directory: ${{ inputs.working-directory }}
      env:
        AUDITAI_CONFIG: ${{ inputs.config }}
        AUDITAI_OUT: ${{ inputs.out }}
        AUDITAI_FAIL_ON: ${{ inputs.fail-on }}
        AUDITAI_BASELINE: ${{ inputs.baseline }}
        AUDITAI_MAX_DROP: ${{ inputs.max-drop }}
      run: bash "${{ github.action_path }}/action/run.sh"

    - name: Upload report artifact
      if: ${{ inputs.upload-artifact == 'true' && always() }}
      uses: actions/upload-artifact@v4
      with:
        name: auditai-report
        path: |
          ${{ inputs.working-directory }}/${{ inputs.out }}/auditai-report.md
          ${{ inputs.working-directory }}/${{ inputs.out }}/auditai-report.json
        if-no-files-found: warn

    - name: Comment on Pull Request
      if: ${{ inputs.comment-on-pr == 'true' && github.event_name == 'pull_request' && always() }}
      uses: actions/github-script@v7
      env:
        REPORT_MD: ${{ inputs.working-directory }}/${{ inputs.out }}/auditai-report.md
        AUDIT_PASSED: ${{ steps.audit.outputs.passed }}
        AUDIT_EXIT: ${{ steps.audit.outputs.exit-code }}
      with:
        github-token: ${{ inputs.github-token }}
        script: |
          const fs = require('fs');
          const path = process.env.REPORT_MD;
          let body;
          if (fs.existsSync(path)) {
            body = fs.readFileSync(path, 'utf8');
          } else {
            body = `## 🛡️ AuditAI Report\n\n_No report file found._\n\nexit_code=${process.env.AUDIT_EXIT}`;
          }
          // Marker so we can update the same comment on re-runs
          const marker = '<!-- auditai-report -->';
          body = `${marker}\n${body}\n\n<sub>exit_code=${process.env.AUDIT_EXIT} · passed=${process.env.AUDIT_PASSED}</sub>`;

          const { owner, repo } = context.repo;
          const issue_number = context.issue.number;
          const { data: comments } = await github.rest.issues.listComments({
            owner, repo, issue_number, per_page: 100,
          });
          const existing = comments.find(c => c.body && c.body.includes(marker));
          if (existing) {
            await github.rest.issues.updateComment({
              owner, repo, comment_id: existing.id, body,
            });
          } else {
            await github.rest.issues.createComment({
              owner, repo, issue_number, body,
            });
          }

    - name: Fail job if audit failed
      if: always()
      shell: bash
      env:
        AUDIT_EXIT: ${{ steps.audit.outputs.exit-code }}
      run: |
        code="${AUDIT_EXIT:-3}"
        if [[ "$code" == "0" ]]; then
          exit 0
        fi
        echo "AuditAI gate failed with exit code $code"
        exit "$code"