conformance-weekly #22
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: conformance-weekly | |
| # Tracks the newest published @modelcontextprotocol/conformance; the | |
| # on:pull_request pipeline pins an exact version instead. Also publishes | |
| # score badges to the orphan `badges` branch, consumed by the README. | |
| on: | |
| schedule: | |
| - cron: '0 6 * * 1' # Mondays 06:00 UTC | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| issues: write | |
| jobs: | |
| server: | |
| name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - spec-version: '2025-11-25' | |
| dist-tag: latest | |
| path: '/' | |
| baseline: conformance-baseline-2025-11-25.yml | |
| - spec-version: '2026-07-28' | |
| dist-tag: alpha | |
| path: '/' | |
| baseline: conformance-baseline-2026-07-28.yml | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/setup-node@v7 | |
| with: | |
| node-version: '22' | |
| - run: composer install --prefer-dist --no-progress --no-interaction | |
| - name: Start conformance server | |
| run: | | |
| mkdir -p tests/Conformance/sessions tests/Conformance/logs | |
| chmod -R 777 tests/Conformance/sessions tests/Conformance/logs | |
| docker compose -f tests/Conformance/Fixtures/docker-compose.yml up -d | |
| sleep 5 | |
| - name: Run conformance tests | |
| working-directory: ./tests/Conformance | |
| run: >- | |
| npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} server | |
| --url http://localhost:8000${{ matrix.path }} | |
| --suite all | |
| --spec-version ${{ matrix.spec-version }} | |
| --expected-failures ${{ matrix.baseline }} | |
| --output-dir results | |
| - name: Generate score badge | |
| if: always() | |
| run: php tests/Conformance/score.php server ${{ matrix.spec-version }} | |
| - name: Show docker logs on failure | |
| if: failure() | |
| run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs | |
| - name: Upload conformance results | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: conformance-server-results-${{ matrix.spec-version }} | |
| path: | | |
| tests/Conformance/logs | |
| tests/Conformance/results | |
| - name: Upload score badge | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: badge-server-${{ matrix.spec-version }} | |
| path: tests/Conformance/server-conformance-${{ matrix.spec-version }}.json | |
| client: | |
| name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - spec-version: '2025-11-25' | |
| dist-tag: latest | |
| baseline: conformance-baseline-2025-11-25.yml | |
| - spec-version: '2026-07-28' | |
| dist-tag: alpha | |
| baseline: conformance-baseline-2026-07-28.yml | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: shivammathur/setup-php@v2 | |
| with: | |
| php-version: '8.4' | |
| coverage: none | |
| - uses: actions/setup-node@v7 | |
| with: | |
| node-version: '22' | |
| - run: composer install --prefer-dist --no-progress --no-interaction | |
| - run: mkdir -p tests/Conformance/logs | |
| - name: Run conformance tests | |
| working-directory: ./tests/Conformance | |
| run: >- | |
| npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} client | |
| --command "php ${{ github.workspace }}/tests/Conformance/client.php" | |
| --suite all | |
| --spec-version ${{ matrix.spec-version }} | |
| --expected-failures ${{ matrix.baseline }} | |
| --output-dir results | |
| - name: Generate score badge | |
| if: always() | |
| run: php tests/Conformance/score.php client ${{ matrix.spec-version }} | |
| - name: Upload conformance results | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: conformance-client-results-${{ matrix.spec-version }} | |
| path: | | |
| tests/Conformance/logs | |
| tests/Conformance/results | |
| - name: Upload score badge | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: badge-client-${{ matrix.spec-version }} | |
| path: tests/Conformance/client-conformance-${{ matrix.spec-version }}.json | |
| notify: | |
| name: Open issue on failure | |
| runs-on: ubuntu-latest | |
| needs: [server, client] | |
| if: failure() && github.event_name == 'schedule' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GH_REPO: ${{ github.repository }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| steps: | |
| - name: File or comment tracking issue | |
| run: | | |
| existing=$(gh issue list --label conformance-weekly --state open --json number --jq '.[0].number // empty') | |
| if [ -n "$existing" ]; then | |
| gh issue comment "$existing" --body "New failure on $(date -u +%FT%TZ): $RUN_URL" | |
| else | |
| gh issue create \ | |
| --title '[conformance] Weekly conformance run failed' \ | |
| --label conformance-weekly \ | |
| --body "Weekly conformance against \`@modelcontextprotocol/conformance@latest\` failed. | |
| - Run: $RUN_URL | |
| - Triggered: $(date -u +%FT%TZ) | |
| Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to \`tests/Conformance/conformance-baseline-2025-11-25.yml\`." | |
| fi | |
| publish: | |
| name: Publish conformance badges | |
| runs-on: ubuntu-latest | |
| needs: [server, client] | |
| if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/php-sdk' }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| pattern: badge-* | |
| path: badges-in | |
| merge-multiple: true | |
| - name: Publish to badges branch | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| if git ls-remote --exit-code --heads origin badges >/dev/null 2>&1; then | |
| git fetch origin badges | |
| git worktree add badges-wt badges | |
| else | |
| git worktree add --detach badges-wt | |
| git -C badges-wt checkout --orphan badges | |
| git -C badges-wt rm -rf --quiet . >/dev/null 2>&1 || true | |
| fi | |
| rm -f badges-wt/server-conformance.json badges-wt/client-conformance.json | |
| cp badges-in/*.json badges-wt/ | |
| cd badges-wt | |
| git add -A | |
| if git diff --cached --quiet; then | |
| echo "Conformance scores unchanged." | |
| else | |
| git commit -m "Update conformance score badges" | |
| git push origin badges | |
| fi |