CI #71
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # Two jobs, two purposes: | |
| # unit-tests -> the PR GATE. Fast, free, deterministic, no LLM keys. | |
| # Runs on every pull request and push to main. | |
| # faithfulness-eval -> LLM-based RAGAS eval. Runs ONLY nightly or on manual | |
| # trigger, never on PRs, so it does not burn LLM quota | |
| # on every push (and won't fail PRs on a stray 429). | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| schedule: | |
| - cron: "0 3 * * *" # nightly 03:00 UTC | |
| workflow_dispatch: # manual "Run workflow" button | |
| jobs: | |
| # ── Gate: free, deterministic offline tests on every PR/push ────────────── | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: pip-${{ hashFiles('requirements.txt') }} | |
| - run: pip install -r requirements.txt | |
| - run: pytest tests/unit -q | |
| # ── Faithfulness eval: nightly / manual only (no quota burn on PRs) ─────── | |
| faithfulness-eval: | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| # Free-tier LLM eval is informational, not a hard gate — don't fail the | |
| # workflow on a flaky 429/413. The deterministic unit-tests job is the gate. | |
| continue-on-error: true | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: pip-${{ hashFiles('requirements.txt') }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/huggingface | |
| key: hf-models-bge-small-reranker-base | |
| - run: pip install -r requirements.txt | |
| - name: Run faithfulness evaluation | |
| env: | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} # generation | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} # eval judge | |
| TOKENIZERS_PARALLELISM: "false" | |
| run: python app/services/evaluation/eval.py --subset 5 |