Guardrails/.github/workflows/api-conformance-docs.yml at develop · NVIDIA-NeMo/Guardrails · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
# Docs conformance: diffs the build-time OpenAPI spec with its Fern overlay
# against the upstream OpenAI spec.  Runs on PRs that touch the spec and
# weekly to surface upstream drift.
#
# PR checks use a pinned upstream ref for reproducibility.
# Weekly checks track master to surface upstream drift.

name: "API Conformance: Docs"

on:
  pull_request:
    paths:
      - "nemoguardrails/**"
      - "pyproject.toml"
      - "uv.lock"
      - "scripts/export_openapi.py"
      - "fern/openapi-overlay.yml"
      - "fern/generators.yml"
      - "fern/fern.config.json"
      - "Makefile"
      - "scripts/openai_coverage.py"
      - ".github/workflows/api-conformance-docs.yml"

  schedule:
    - cron: "23 9 * * 1"

  workflow_dispatch:

permissions:
  contents: read

env:
  OASDIFF_VERSION: "1.23.0"
  OPENAI_SPEC_PIN: "f9400172ebe08522ab228b771d885e3bd5456e42"
  OPENAI_SPEC_URL: "https://raw.githubusercontent.com/openai/openai-openapi"

jobs:
  docs:
    name: Generated Fern API vs OpenAI spec
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - uses: ./.github/actions/setup-uv
      - run: uv sync --locked --group dev

      - name: Set up Node.js
        uses: actions/setup-node@v7
        with:
          node-version: "22"
          package-manager-cache: false

      - name: Generate published API contract
        run: |
          make docs-fern-generate-openapi
          FERN_VERSION=$(node -p "require('./fern/fern.config.json').version")
          cd fern
          npx --yes "fern-api@${FERN_VERSION}" export "$RUNNER_TEMP/openapi-current.yml"

      - name: Install oasdiff
        run: |
          go install github.com/oasdiff/oasdiff@v${{ env.OASDIFF_VERSION }}
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH

      - name: Fetch OpenAI spec
        run: |
          if [ "${{ github.event_name }}" = "schedule" ]; then
            REF="master"
          else
            REF="${{ env.OPENAI_SPEC_PIN }}"
          fi
          curl -fsSL "${{ env.OPENAI_SPEC_URL }}/${REF}/openapi.yaml" \
            -o /tmp/openai-spec.yml

      - name: Check for breaking changes
        if: github.event_name == 'pull_request'
        run: |
          # actions/checkout uses GitHub's test merge, whose first parent is
          # the actual target revision; the PR event's base SHA can be stale.
          BASE_REF=$(git rev-parse HEAD^1)
          # Initialize the generated baseline only when runtime sources are unchanged.
          # The legacy handwritten schema has drifted from the implementation.
          if ! git cat-file -e "$BASE_REF:scripts/export_openapi.py" 2>/dev/null &&
             git diff --quiet "$BASE_REF" HEAD -- nemoguardrails pyproject.toml uv.lock; then
            echo "::notice::Initializing generated OpenAPI baseline from unchanged runtime sources."
          else
            BASE_DIR="$RUNNER_TEMP/openapi-base"
            trap 'git worktree remove --force "$BASE_DIR"' EXIT
            git -c core.hooksPath=/dev/null worktree add --detach "$BASE_DIR" "$BASE_REF"
            if [ -f "$BASE_DIR/scripts/export_openapi.py" ]; then
              (cd "$BASE_DIR" && env -u FERN_TOKEN -u DOCS_FERN_TOKEN \
                UV_PROJECT_ENVIRONMENT="$BASE_DIR/.venv" \
                uv run --locked --extra server python scripts/export_openapi.py)
            fi
            FERN_VERSION=$(node -p "require('./fern/fern.config.json').version")
            (cd "$BASE_DIR/fern" && npx --yes "fern-api@${FERN_VERSION}" \
              export "$RUNNER_TEMP/openapi-base.yml")
            uv run --locked python scripts/openai_coverage.py \
              --openai-spec /tmp/openai-spec.yml --check-breaking \
              --guardrails-spec "$RUNNER_TEMP/openapi-current.yml" \
              --base-spec "$RUNNER_TEMP/openapi-base.yml" \
              --base-ref "$BASE_REF"
          fi

      - name: Check conformance
        run: |
          uv run --locked python scripts/openai_coverage.py \
            --openai-spec /tmp/openai-spec.yml \
            --guardrails-spec "$RUNNER_TEMP/openapi-current.yml" \
            --summary "$GITHUB_STEP_SUMMARY"