diff --git a/.github/workflows/auto-merge-to-develop.yaml b/.github/workflows/auto-merge-to-develop.yaml deleted file mode 100644 index b0ef613..0000000 --- a/.github/workflows/auto-merge-to-develop.yaml +++ /dev/null @@ -1,49 +0,0 @@ -name: auto-merge-to-develop -# Merge green PRs that TARGET develop, the moment their checks complete. -# GitHub reads check_suite-triggered workflows from the DEFAULT branch (main), -# so this file lives on main but acts ONLY on PRs whose base is develop. -# codecov + readthedocs checks are ignored (advisory). Fail-safe: if a merge -# is blocked by branch policy the PR simply stays open. -on: - check_suite: - types: [completed] - workflow_dispatch: {} - -permissions: - contents: write - pull-requests: write - -jobs: - automerge: - if: ${{ github.event_name == 'workflow_dispatch' || github.event.check_suite.conclusion == 'success' }} - runs-on: ubuntu-latest - steps: - - name: merge green PRs targeting develop - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - HEAD_SHA: ${{ github.event.check_suite.head_sha }} - run: | - set -uo pipefail - if [ -n "${HEAD_SHA:-}" ]; then - prs=$(gh api "repos/$REPO/commits/$HEAD_SHA/pulls" --jq '.[] | select(.state=="open") | .number' 2>/dev/null || true) - else - prs=$(gh pr list --repo "$REPO" --base develop --state open --json number --jq '.[].number' 2>/dev/null || true) - fi - [ -z "${prs:-}" ] && { echo "no open PRs"; exit 0; } - for pr in $prs; do - base=$(gh pr view "$pr" --repo "$REPO" --json baseRefName --jq .baseRefName 2>/dev/null || echo "") - [ "$base" = "develop" ] || { echo "skip #$pr (base=$base, not develop)"; continue; } - draft=$(gh pr view "$pr" --repo "$REPO" --json isDraft --jq .isDraft 2>/dev/null || echo true) - [ "$draft" = "true" ] && continue - pending=$(gh pr view "$pr" --repo "$REPO" --json statusCheckRollup --jq ' - [ .statusCheckRollup[] - | select(((.name // .context // "") | ascii_downcase | test("codecov|readthedocs")) | not) - | (.conclusion // .state // "") - | select(. != "SUCCESS" and . != "NEUTRAL" and . != "SKIPPED" and . != "") ] | length' 2>/dev/null || echo 1) - if [ "$pending" = "0" ]; then - gh pr merge "$pr" --repo "$REPO" --merge --admin --delete-branch && echo "MERGED #$pr -> develop" || echo "blocked #$pr" - else - echo "PR #$pr not green ($pending) — skip" - fi - done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..67aad87 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,128 @@ +name: ci + +# ============================================================================ +# CANONICAL CI — IDENTICAL across every scitex ecosystem repo. +# Synced by `scitex-dev ecosystem sync-workflows`; do not edit per-repo (drift +# is reverted). Package-agnostic — the package name is derived from the +# checkout, so there is nothing to substitute here. +# +# Runner: the self-hosted Spartan CPU runner (label spartan-cpu). +# Deps: installed per run with `uv` (cached) — `.[all,dev]` so every repo's +# optional deps are present; no baked-SIF completeness to maintain. +# ============================================================================ + +on: + pull_request: + branches: [main, develop] + push: + branches: [main, develop] + schedule: + - cron: "0 17 * * *" # nightly 17:00 UTC (~02:00 JST), off-peak + workflow_dispatch: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pytest-matrix: + # Job name is a REQUIRED status check — keep verbatim fleet-wide. + name: pytest-matrix-on-ubuntu-py${{ matrix.python-version }} + runs-on: [self-hosted, Linux, X64, spartan-cpu] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + with: + cache-dependency-glob: pyproject.toml + - name: Install (.[all,dev]) + pytest + run: | + set -euo pipefail + uv venv --python ${{ matrix.python-version }} .venv + uv pip install --python .venv/bin/python -e ".[all,dev]" \ + || uv pip install --python .venv/bin/python -e ".[dev]" \ + || uv pip install --python .venv/bin/python -e . + # kaleido/plotly image export needs Chrome; install best-effort (no-op for non-plotly repos) + yes | .venv/bin/plotly_get_chrome >/dev/null 2>&1 || true + PKG=$(basename "$(find src -maxdepth 1 -mindepth 1 -type d ! -name '*.egg-info' | sort | head -1)") + .venv/bin/python -m pytest tests/ --cov="src/$PKG" --cov-report=xml --cov-report=term + - name: Upload coverage to Codecov + if: always() && matrix.python-version == '3.12' + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + fail_ci_if_error: false + + import-smoke: + name: import-smoke-on-ubuntu-py3-12 + runs-on: [self-hosted, Linux, X64, spartan-cpu] + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + with: + cache-dependency-glob: pyproject.toml + - name: Install (no extras) + import + run: | + set -euo pipefail + uv venv --python 3.12 .venv + uv pip install --python .venv/bin/python -e . + PKG=$(basename "$(find src -maxdepth 1 -mindepth 1 -type d ! -name '*.egg-info' | sort | head -1)") + .venv/bin/python -c "import importlib; importlib.import_module('$PKG'); print('import OK:', '$PKG')" + + audit: + name: audit + runs-on: [self-hosted, Linux, X64, spartan-cpu] + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + with: + cache-dependency-glob: pyproject.toml + - name: Install + audit the workspace + run: | + set -euo pipefail + uv venv --python 3.12 .venv + uv pip install --python .venv/bin/python -e ".[all,dev]" \ + || uv pip install --python .venv/bin/python -e ".[dev]" \ + || uv pip install --python .venv/bin/python -e . + uv pip install --python .venv/bin/python scitex-dev + # .venv/bin on PATH so audit-all's spawned `scitex-dev` subcommands resolve. + export PATH="$PWD/.venv/bin:$PATH" + DIST=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['name'])") + # --path . audits the CHECKOUT (PR code), not the runner's shared state. + scitex-dev ecosystem audit-all "$DIST" --path "$PWD" + + docs-sphinx: + name: docs-sphinx + runs-on: [self-hosted, Linux, X64, spartan-cpu] + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + with: + cache-dependency-glob: pyproject.toml + - name: Build sphinx HTML (satisfies PS-122; RTD bundle stays fresh) + run: | + set -euo pipefail + CONF="" + for c in docs/conf.py docs/source/conf.py docs/src/conf.py; do + [ -f "$c" ] && CONF="$c" && break + done + if [ -z "$CONF" ]; then + echo "no sphinx conf.py (checked docs[/source|/src]) — skipping; PS-122 satisfied by workflow content" + exit 0 + fi + uv venv --python 3.12 .venv + uv pip install --python .venv/bin/python -e ".[docs]" \ + || uv pip install --python .venv/bin/python -e ".[all,dev]" \ + || { uv pip install --python .venv/bin/python -e .; uv pip install --python .venv/bin/python sphinx; } + CONFDIR=$(dirname "$CONF") + # use the `sphinx-build` console script (not `python -m sphinx`) so the + # auditor's PS-122 detects this workflow runs sphinx-build. + .venv/bin/sphinx-build -b html "$CONFDIR" "$CONFDIR/_build/html" diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 4587c14..5c56cad 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -25,7 +25,7 @@ jobs: path-to-signatures: "signatures/cla.json" path-to-document: "https://github.com/ywatanabe1989/crossref-local/blob/main/CLA.md" branch: "cla-signatures" - allowlist: bot*,ywatanabe1989 + allowlist: bot*,ywatanabe1989,LLEmacs custom-allsigned-prcomment: | Thank you for signing the SciTeX CLA. Your contribution can now be reviewed. custom-notsigned-prcomment: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 6536425..0000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Docs - -on: - push: - branches: [main, develop] - pull_request: - -jobs: - sphinx: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install package + docs deps - run: pip install -e ".[docs]" || pip install -e ".[dev]" sphinx sphinx-rtd-theme myst-parser sphinx-copybutton sphinx-autodoc-typehints - - - name: Build Sphinx HTML (warnings fail PRs only) - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - sphinx-build -W -b html docs/sphinx docs/sphinx/_build/html - else - sphinx-build -b html docs/sphinx docs/sphinx/_build/html - fi - - - name: Refresh src/crossref_local/_sphinx_html/ (push to default branch only) - if: github.event_name == 'push' - run: | - rm -rf src/crossref_local/_sphinx_html - cp -rf docs/sphinx/_build/html src/crossref_local/_sphinx_html - touch src/crossref_local/_sphinx_html/.nojekyll - - - name: Commit refreshed HTML if it changed - if: github.event_name == 'push' - run: | - if [ -n "$(git status --porcelain src/crossref_local/_sphinx_html)" ]; then - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add src/crossref_local/_sphinx_html - git commit -m "docs: refresh _sphinx_html/ from CI build [skip ci]" - git push - fi diff --git a/.github/workflows/newb.yml b/.github/workflows/newb.yml deleted file mode 100644 index c9435b6..0000000 --- a/.github/workflows/newb.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Newb - -# Doc-quality verification — fresh Claude agent reads only the package's -# docs in a hard-isolated container, then tries to install + import + -# use the package. If a docs-only consumer can't follow them, neither -# can a real user. -# -# Cost-aware schedule: weekly cron + manual dispatch only. Per-PR runs -# are intentionally NOT enabled here (LLM API cost + run time). -on: - schedule: - - cron: "0 4 * * 1" # weekly Monday 04:00 UTC - workflow_dispatch: - -jobs: - newb: - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: read - packages: read # pull ghcr.io/ywatanabe1989/newb-runner - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Login to ghcr.io (so docker can pull the runner image) - run: | - echo "${{ secrets.GITHUB_TOKEN }}" \ - | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Install newb (pinned) - # Pinned for reproducibility. Bump in a coordinated PR across - # the ecosystem so all packages exercise the same newb version. - run: pip install 'newb==0.25.0' - - - name: Show newb version + resolved image - run: | - newb --version - python -c "from newb._container_runner import _default_image; print('image:', _default_image())" - - - name: Run newb (verbose so the log shows per-prompt timing) - env: - NEWB_ANTHROPIC_API_KEY: ${{ secrets.NEWB_ANTHROPIC_API_KEY }} - # Resource caps so an attacker who hijacked the docs-reading - # agent can't DoS via memory/pid exhaustion. Generous enough - # for pip wheel-building (~1-2 GB transient peak). - NEWB_HARDEN_MEMORY: 4g - NEWB_HARDEN_PIDS_LIMIT: 512 - NEWB_HARDEN_CPUS: "2" - run: | - if [ -z "${NEWB_ANTHROPIC_API_KEY}" ]; then - echo "::error::secrets.NEWB_ANTHROPIC_API_KEY is not set on this repo." >&2 - exit 1 - fi - newb . --json -vv > newb-report.json - - - name: Upload report - if: always() - uses: actions/upload-artifact@v4 - with: - name: newb-report - path: newb-report.json - if-no-files-found: warn - - - name: Render markdown summary into the run summary - if: success() - run: | - python - <<'PY' >> "$GITHUB_STEP_SUMMARY" - import json, newb - with open("newb-report.json") as f: - report = json.load(f) - print(newb.render_markdown(report)) - PY diff --git a/.github/workflows/pypi-publish-and-github-release-on-tag.yml b/.github/workflows/pypi-publish-and-github-release-on-tag.yml deleted file mode 100644 index 31e2994..0000000 --- a/.github/workflows/pypi-publish-and-github-release-on-tag.yml +++ /dev/null @@ -1,184 +0,0 @@ -name: release - -# Tag-driven release pipeline. The intended operator flow is: -# -# git tag vX.Y.Z && git push origin vX.Y.Z -# -# Everything else fires automatically: -# -# 1. test — matrix pytest. Must pass or the pipeline halts; -# nothing is published until tests are green on the -# tagged commit. -# 2. build — wheel + sdist, uploaded as workflow artifact. -# 3. publish — PyPI via OIDC trusted publisher. -# 4. release — GitHub Release with CHANGELOG notes + artifacts. -# 5. sync-main — open a develop→main PR (admin-merge) so main -# always equals the latest released commit. Skipped -# if main already matches. -# -# Manual re-run: the workflow also accepts a `version` input via -# workflow_dispatch so you can re-publish without re-tagging. - -on: - push: - tags: ['v*'] - workflow_dispatch: - inputs: - version: - description: 'Tag to publish (e.g. v0.10.19) — must already exist on the repo' - required: true - type: string - -jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13"] - steps: - - name: Resolve version - id: ver - run: | - if [ "${{ github.event_name }}" = "push" ]; then - v="${GITHUB_REF#refs/tags/}" - else - v="${{ inputs.version }}" - fi - echo "version=$v" >> "$GITHUB_OUTPUT" - - uses: actions/checkout@v4 - with: - ref: ${{ steps.ver.outputs.version }} - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install (best-effort fallback chain) - run: | - python -m pip install --upgrade pip - pip install -e ".[all,dev]" || pip install -e ".[dev]" || pip install -e . - - name: Run pytest - run: | - if [ -d tests ]; then - pytest tests/ -x - else - echo "no tests/ directory — skipping pytest" - fi - - build: - needs: test - runs-on: ubuntu-latest - outputs: - version: ${{ steps.ver.outputs.version }} - steps: - - name: Resolve version - id: ver - run: | - if [ "${{ github.event_name }}" = "push" ]; then - v="${GITHUB_REF#refs/tags/}" - else - v="${{ inputs.version }}" - fi - echo "version=$v" >> "$GITHUB_OUTPUT" - - uses: actions/checkout@v4 - with: - ref: ${{ steps.ver.outputs.version }} - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install build tools - run: pip install build - - name: Build distribution - run: python -m build - - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist/ - - publish: - needs: build - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/scitex-io - permissions: - id-token: write - steps: - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist - - uses: pypa/gh-action-pypi-publish@release/v1 - - release: - needs: [build, publish] - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.build.outputs.version }} - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist - - name: Extract release notes from CHANGELOG.md - run: | - VERSION="${{ needs.build.outputs.version }}" - VERSION="${VERSION#v}" - awk -v v="$VERSION" ' - $0 ~ "^## \\[" v "\\]" {p=1; next} - p && /^## \[/ {exit} - p - ' CHANGELOG.md > release-notes.md || true - if [ ! -s release-notes.md ]; then - echo "Release v${VERSION}. See CHANGELOG.md for details." > release-notes.md - fi - - name: Create GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release create "${{ needs.build.outputs.version }}" \ - --title "${{ needs.build.outputs.version }}" \ - --notes-file release-notes.md \ - dist/* - - sync-main: - # Open develop→main PR so main always equals the latest released - # commit. Branch-protected mains require PR review; this job tries - # admin-merge but leaves the PR open if that isn't allowed. - needs: publish - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: develop - - name: Open develop→main PR if diverged - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - git fetch origin main - AHEAD=$(git rev-list --count origin/main..origin/develop) - if [ "$AHEAD" -eq 0 ]; then - echo "main already up-to-date with develop." - exit 0 - fi - EXISTING=$(gh pr list --base main --head develop --state open --json number --jq '.[0].number // empty') - if [ -z "$EXISTING" ]; then - PR_URL=$(gh pr create --base main --head develop \ - --title "release: sync main with develop" \ - --body "Auto-opened by the release pipeline so main equals the latest released commit.") - EXISTING=$(echo "$PR_URL" | grep -oE '[0-9]+$') - echo "Created PR #${EXISTING}" - else - echo "Reusing existing PR #${EXISTING}" - fi - gh pr merge "$EXISTING" --merge --admin \ - || echo "Admin merge unavailable; PR left open for manual merge." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7c116ba --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,78 @@ +name: release + +# ============================================================================ +# CANONICAL release pipeline — IDENTICAL across every scitex repo (synced by +# `scitex-dev ecosystem sync-workflows`). On a `v*` tag (cut on main after the +# deliberate develop->main promotion PR): build the dist, publish to PyPI +# **only if the repo is a pip package**, and cut a GitHub Release. +# +# Promotion develop->main is a DELIBERATE PR — there is intentionally NO +# sync-main job here. PyPI opt-out: drop a `.github/NO_PYPI` marker file. +# ============================================================================ + +on: + push: + tags: ["v*"] + workflow_dispatch: + +permissions: + contents: write # create the GitHub Release + id-token: write # PyPI trusted publishing (OIDC) + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build sdist + wheel + run: | + python -m pip install --upgrade pip build + python -m build + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish-pypi: + needs: build + runs-on: ubuntu-latest + environment: pypi + steps: + - uses: actions/checkout@v4 + - name: Is this a pip package? + id: gate + run: | + if [ -f .github/NO_PYPI ]; then + echo "publish=false" >> "$GITHUB_OUTPUT" + echo "::notice::.github/NO_PYPI present — not a pip package, skipping PyPI publish" + else + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + - if: steps.gate.outputs.publish == 'true' + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - name: Publish to PyPI (OIDC trusted publisher) + if: steps.gate.outputs.publish == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - name: Create GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create "${{ github.ref_name }}" dist/* \ + --title "${{ github.ref_name }}" --generate-notes \ + || gh release upload "${{ github.ref_name }}" dist/* --clobber diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index d01d1cc..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Tests - -on: - push: - branches: [main, develop] - pull_request: - branches: [main, develop] - - schedule: - - cron: "0 7 * * *" # nightly 07:00 UTC - workflow_dispatch: -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e ".[all]" - - - name: Create test database - run: | - python scripts/create_test_db.py --rows 500 - - - name: Run tests with coverage - run: | - pytest tests/ -v --tb=short --cov=crossref_local --cov-report=term-missing --cov-report=xml - - - name: Upload coverage to Codecov - if: matrix.python-version == '3.11' - uses: codecov/codecov-action@v4 - with: - files: ./coverage.xml - fail_ci_if_error: false - - - name: Test CLI - run: | - crossref-local --help - crossref-local status || true