promptfoo-action/action.yml at main · toptal/promptfoo-action · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
name: 'Test LLM outputs'
description: 'Run promptfoo evaluations on your prompts. Works with pull requests, pushes, and manual triggers (workflow_dispatch).'
runs:
  using: 'node20'
  main: 'dist/index.js'
# For workflow_dispatch events, you can use the following inputs in your workflow:
# - inputs.files: Newline-separated list of files to evaluate (optional)
# - inputs.base: Base branch/commit to compare against (default: HEAD~1) (optional)
#
# Example workflow_dispatch configuration:
# on:
#   workflow_dispatch:
#     inputs:
#       files:
#         description: 'Files to evaluate (leave empty to auto-detect)'
#         required: false
#         type: string
#       base:
#         description: 'Base branch/commit to compare against'
#         required: false
#         default: 'HEAD~1'
#         type: string
inputs:
  github-token:
    description: 'Github Token, used to add comments to PRs and write workflow summaries'
    required: true
  prompts:
    description: 'List of prompt files to watch (glob patterns supported, one per line)'
    required: false
  config:
    description: 'Path to a promptfoo config file'
    required: true
  cache-path:
    description: 'Path to cache directory'
    required: false
  openai-api-key:
    description: 'OpenAI API Key'
    required: false
  azure-api-key:
    description: 'Azure API key'
    required: false
  anthropic-api-key:
    description: 'Anthropic API key'
    required: false
  huggingface-api-key:
    description: 'Huggingface API key'
    required: false
  aws-access-key-id:
    description: 'AWS Access Key ID'
    required: false
  aws-secret-access-key:
    description: 'AWS Secret Access Key'
    required: false
  replicate-api-key:
    description: 'Replicate API key'
    required: false
  palm-api-key:
    description: 'Palm API key'
    required: false
  vertex-api-key:
    description: 'Google vertex API key'
    required: false
  cohere-api-key:
    description: 'Cohere API key'
    required: false
  mistral-api-key:
    description: 'Mistral API key'
    required: false
  groq-api-key:
    description: 'Groq API key'
    required: false
  promptfoo-version:
    description: 'Version of promptfoo to use'
    required: false
    default: 'latest'
  working-directory:
    description: 'The working directory to run promptfoo in'
    required: false
    default: '.'
  no-share:
    description: 'Do not share the promptfoo result'
    required: false
    default: 'false'
  use-config-prompts:
    description: 'Use prompts from config file'
    required: false
    default: 'false'
  env-files:
    description: 'Comma-separated list of .env files to load (e.g. ".env,.env.test.local")'
    required: false
    default: ''
  fail-on-threshold:
    description: 'Fail the action if the evaluation success rate is below this percentage (0-100)'
    required: false
    default: ''
  debug:
    description: 'Enable debug logging for troubleshooting'
    required: false
    default: 'false'
  max-concurrency:
    description: 'Maximum number of concurrent API calls (default: 4)'
    required: false
    default: '4'
  no-table:
    description: 'Run promptfoo with --no-table flag to keep output minimal'
    required: false
    default: 'false'
  no-progress-bar:
    description: 'Run promptfoo with --no-progress-bar flag to keep output minimal'
    required: false
    default: 'false'
  no-cache:
    description: 'Run promptfoo with --no-cache flag to avoid reading or writing results to the disk cache'
    required: false
    default: 'false'
  disable-comment:
    description: 'Disable posting comments to the PR'
    required: false
    default: 'false'
  workflow-files:
    description: 'Files to evaluate in workflow_dispatch mode (newline-separated list). If not specified, falls back to workflow inputs or auto-detection.'
    required: false
    default: ''
  workflow-base:
    description: 'Base ref to compare against in workflow_dispatch mode (e.g., main, HEAD~1). If not specified, falls back to workflow inputs or HEAD~1.'
    required: false
    default: ''
  force-run:
    description: 'Force evaluation to run even if no files changed'
    required: false
    default: 'false'
branding:
  icon: 'box'
  color: 'yellow'