action.yml

name: 'Test LLM outputs'
description: 'Run promptfoo evaluations on your prompts. Works with pull requests, pushes, and manual triggers (workflow_dispatch).'
runs:
  using: 'node24'
  main: 'dist/index.js'
# For workflow_dispatch events, you can use the following inputs in your workflow:
# - inputs.files: Newline-separated changed-file list to consider (optional)
# - inputs.base: Base branch/commit to compare against (default: HEAD~1) (optional)
#
# Example workflow_dispatch configuration:
# on:
#   workflow_dispatch:
#     inputs:
#       files:
#         description: 'Changed files to consider (leave empty to auto-detect)'
#         required: false
#         type: string
#       base:
#         description: 'Base branch/commit to compare against'
#         required: false
#         default: 'HEAD~1'
#         type: string
inputs:
  github-token:
    description: 'GitHub token used to list pull request files and post pull request comments'
    required: true
  prompts:
    description: 'List of prompt files to watch (glob patterns supported, one per line)'
    required: false
  config:
    description: 'Path to a Promptfoo config file'
    required: true
  cache-path:
    description: 'Path to the Promptfoo disk cache directory'
    required: false
  openai-api-key:
    description: 'OpenAI API Key. Falls back to OPENAI_API_KEY environment variable if not provided.'
    required: false
  azure-api-key:
    description: 'Azure API key. Falls back to AZURE_OPENAI_API_KEY environment variable if not provided.'
    required: false
  anthropic-api-key:
    description: 'Anthropic API key. Falls back to ANTHROPIC_API_KEY environment variable if not provided.'
    required: false
  huggingface-api-key:
    description: 'Hugging Face API key. Falls back to HF_API_TOKEN environment variable if not provided.'
    required: false
  aws-access-key-id:
    description: 'AWS Access Key ID. Falls back to AWS_ACCESS_KEY_ID environment variable if not provided.'
    required: false
  aws-secret-access-key:
    description: 'AWS Secret Access Key. Falls back to AWS_SECRET_ACCESS_KEY environment variable if not provided.'
    required: false
  replicate-api-key:
    description: 'Replicate API key. Falls back to REPLICATE_API_KEY environment variable if not provided.'
    required: false
  palm-api-key:
    description: 'Palm API key. Falls back to PALM_API_KEY environment variable if not provided.'
    required: false
  vertex-api-key:
    description: 'Google Vertex API key. Falls back to VERTEX_API_KEY environment variable if not provided.'
    required: false
  cohere-api-key:
    description: 'Cohere API key. Falls back to COHERE_API_KEY environment variable if not provided.'
    required: false
  mistral-api-key:
    description: 'Mistral API key. Falls back to MISTRAL_API_KEY environment variable if not provided.'
    required: false
  groq-api-key:
    description: 'Groq API key. Falls back to GROQ_API_KEY environment variable if not provided.'
    required: false
  promptfoo-version:
    description: 'Version of promptfoo to use'
    required: false
    default: 'latest'
  working-directory:
    description: 'The working directory to run promptfoo in'
    required: false
    default: '.'
  no-share:
    description: 'Pass --no-share and override config-level sharing'
    required: false
    default: 'false'
  use-config-prompts:
    description: 'Use prompts from config file'
    required: false
    default: 'false'
  env-files:
    description: 'Comma-separated list of .env files to load (e.g. ".env,.env.test.local")'
    required: false
    default: ''
  fail-on-threshold:
    description: 'Fail the action if the evaluation success rate is below this percentage (0-100)'
    required: false
    default: ''
  debug:
    description: 'Reserved compatibility input; use GitHub Actions step debug logging to display debug messages'
    required: false
    default: 'false'
  max-concurrency:
    description: 'Maximum number of concurrent API calls (default: 4)'
    required: false
    default: '4'
  no-table:
    description: 'Run promptfoo with --no-table flag to keep output minimal'
    required: false
    default: 'false'
  no-progress-bar:
    description: 'Run promptfoo with --no-progress-bar flag to keep output minimal'
    required: false
    default: 'false'
  no-cache:
    description: 'Run promptfoo with --no-cache flag to avoid reading or writing results to the disk cache'
    required: false
    default: 'false'
  disable-comment:
    description: 'Disable posting comments to the PR'
    required: false
    default: 'false'
  workflow-files:
    description: 'Changed-file list for workflow_dispatch mode (newline-separated). If not specified, falls back to workflow inputs or auto-detection.'
    required: false
    default: ''
  workflow-base:
    description: 'Single base ref to compare against in workflow_dispatch mode (e.g., main, HEAD~1). Revision ranges like main..HEAD are not supported. If not specified, falls back to workflow inputs or HEAD~1.'
    required: false
    default: ''
  repeat:
    description: 'Number of times to run each test (must be >= 2). Useful for non-deterministic LLM evals where a single run is noisy.'
    required: false
    default: ''
  repeat-min-pass:
    description: >
      Minimum number of repeated runs each test must pass. Requires repeat to be set.
      Must be <= repeat. Example: with repeat=3 and repeat-min-pass=2, each test must
      pass at least 2 of its 3 runs.
    required: false
    default: ''
  force-run:
    description: 'Force evaluation to run even if no files changed'
    required: false
    default: 'false'
outputs:
  cache-size-mb:
    description: 'Promptfoo cache size in megabytes after the latest metrics check'
  cache-file-count:
    description: 'Number of files in the Promptfoo cache after the latest metrics check'
branding:
  icon: 'box'
  color: 'yellow'