test: apply benchmark regression factor consistently #28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| pull_request: | |
| jobs: | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest, ubuntu-latest] | |
| python: ["3.11", "3.13"] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| cache: pip | |
| - run: python -m pip install -e ".[semantic,dev]" uv | |
| - run: coverage run --branch -m unittest discover -s tests -q | |
| # All first-party modules hold the clean-run 77% baseline. The four | |
| # hardened helpers hold 90. The facade family and its largest gaps have | |
| # separate floors, so one module cannot hide another module's loss. | |
| - run: coverage report --fail-under=77 | |
| - run: coverage report --include="archive_intent.py,archive_store.py,platform_lock.py,schema_migrations.py" --fail-under=90 | |
| - run: coverage report --include="session_search.py,ss_*.py" --fail-under=75 | |
| - run: coverage report --include="ss_adapters.py" --fail-under=74 | |
| - run: coverage report --include="ss_retrieval.py" --fail-under=55 | |
| - run: coverage report --include="ss_cli.py" --fail-under=74 | |
| - run: python tests/run_public_card_quality_eval.py --expected evidence/public-card-quality-v0.1.0.json | |
| - run: python tests/run_public_retrieval_eval.py --mode all --expected evidence/public-retrieval-v0.1.0.json | |
| - run: python tests/benchmark_public.py --sessions 1000 --semantic --expected evidence/public-benchmark-v0.1.0.json | |
| # The published baseline is a macOS hardware measurement. Running its | |
| # model-build limit across unlike hosted runners creates false failures. | |
| if: matrix.os == 'macos-latest' && matrix.python == '3.11' | |
| - run: python tests/run_dashboard_latency_gate.py | |
| - run: uv lock --check | |
| - run: python scripts/check_public_privacy.py | |
| - run: ss demo | |
| package: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-latest, ubuntu-latest] | |
| python: ["3.11", "3.13"] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - run: python -m pip install build "pexpect>=4.9,<5" "wcwidth>=0.2,<1" | |
| - run: python -m build | |
| - run: git worktree add /tmp/session-search-v020 72e26f0 | |
| - run: python -m build --outdir /tmp/session-search-v020-dist /tmp/session-search-v020 | |
| - run: python -m venv /tmp/session-search-install | |
| # Installed on purpose without the semantic extra. The journeys below | |
| # are bounded at 90 seconds and must not reach the network, so they run | |
| # with semantic retrieval unavailable. The test job above covers | |
| # semantic and hybrid retrieval against the published evidence. | |
| - run: /tmp/session-search-install/bin/pip install dist/*.whl | |
| - run: /tmp/session-search-install/bin/ss capabilities | |
| - run: /tmp/session-search-install/bin/ss demo | |
| - run: /tmp/session-search-install/bin/python ${{ github.workspace }}/tests/test_adapter_pipeline.py -q | |
| working-directory: /tmp | |
| - run: /tmp/session-search-install/bin/python ${{ github.workspace }}/tests/run_installed_acceptance.py --ss /tmp/session-search-install/bin/ss | |
| working-directory: /tmp | |
| - run: python ${{ github.workspace }}/tests/run_task_usability_acceptance.py --ss /tmp/session-search-install/bin/ss | |
| working-directory: /tmp | |
| timeout-minutes: 2 | |
| - run: python ${{ github.workspace }}/tests/run_upgrade_acceptance.py --previous-wheel /tmp/session-search-v020-dist/*.whl --candidate-wheel ${{ github.workspace }}/dist/*.whl | |
| if: runner.os == 'macOS' | |
| working-directory: /tmp | |
| timeout-minutes: 2 |