evals.yml

name: Evals

permissions:
  contents: read

on:
  pull_request:
    branches: [ main ]
    paths:
      - 'src/lib/ai.ts'
      - 'src/lib/ci-eval.ts'
      - 'src/lib/eval.ts'
  push:
    branches: [ main ]
    paths:
      - 'src/lib/ai.ts'
      - 'src/lib/ci-eval.ts'
      - 'src/lib/eval.ts'
  workflow_dispatch:

jobs:
  evals:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: '1.2.17'

      - name: Install dependencies
        run: bun install --frozen-lockfile

      - name: Run CI evals
        env:
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
        run: bun run eval:ci
      - name: Upload evals results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: evaluation-results
          path: |
            *.log
            evaluation-*.json
          retention-days: 30