From b8c571113a09fd55be29c45a33b78c451799f6d7 Mon Sep 17 00:00:00 2001 From: zachtheyek Date: Sun, 12 Jul 2026 15:58:57 +0800 Subject: [PATCH 1/5] Finalize hatchling packaging; single-source the version from pyproject MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uncomment and finalize the hatchling build backend (wheel target src/aetherscan) and resolve both stale TODO headers. Fill [project].dependencies from requirements-container.txt's ranges plus the packages the NGC base image provides implicitly (tensorflow[and-cuda]==2.17.*, h5py, hdf5plugin, psutil) — a pip install has no base image to lean on. dev extra gains pre-commit and pytest. Version single-sourcing: pyproject's version (kept a pre-release, 0.9.0.dev0 — the release PR bumps it) is now the only place the version is written; __init__.py reads it back via importlib.metadata with a 0.0.0.dev0 fallback so source-tree/container runs work uninstalled. The pre-existing CUDA '12 :: 12.8' classifier is not in the trove list (tops out at 12 :: 12.6) and fails hatchling validation — replaced with the valid ':: 12' parent. Development Status keeps a TODO knob for the v1 Beta-vs-Production call. gitignore gains python -m build artifacts. Co-Authored-By: Claude Fable 5 --- .gitignore | 4 +++ pyproject.toml | 61 ++++++++++++++++++++++++++++++-------- src/aetherscan/__init__.py | 11 ++++++- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index c6b2cda..74275e7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,10 @@ outputs/ # Container artifacts *.sif +# Python packaging artifacts (python -m build) +dist/ +*.egg-info/ + ### Global gitignore (https://github.com/zachtheyek/dotfiles/blob/master/git/.gitignore_global) diff --git a/pyproject.toml b/pyproject.toml index 386c214..fabb8cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,18 @@ -# TODO: add build backend (uv) -# TODO: How to release as package on PyPI & display on GitHub? How to tag new releases on GitHub? -# [build-system] -# requires = ["hatchling"] -# build-backend = "hatchling.build" -# -# [tool.hatch.build.targets.wheel] -# packages = ["src/aetherscan"] +# Packaging + release process: docs/RELEASE.md (version coupling, CD gates, runbook). +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/aetherscan"] [project] name = "aetherscan" -version = "0.1.0" +# Single source of truth for the package version (src/aetherscan/__init__.py reads it back +# via importlib.metadata). Kept a pre-release between releases; the release PR bumps it to +# the final X.Y.Z, and the CD workflow (.github/workflows/release.yml) enforces that the +# pushed git tag matches it exactly. +version = "0.9.0.dev0" authors = [ { name = "Zach Yek", email = "xiaoxiyek1.5@gmail.com" }, ] @@ -20,7 +23,34 @@ description = "Breakthrough Listen's first end-to-end production-grade DL pipeli readme = "README.md" requires-python = ">=3.10,<3.13" dependencies = [ - # TODO: sync with environment.yml files (include for uv) + # Mirrors requirements-container.txt's ranges — keep the two (and environment.yml / + # aetherscan.def) in lockstep per SECURITY.md's Version Selection Policy. Documented + # ceilings (numpy<2.0, setuptools<81, the NGC TF 2.17 ABI) apply to the package exactly + # as to the container. + "numpy>=1.26,<2.0", + "scipy>=1.13,<1.14", + "scikit-learn>=1.5,<1.6", + "matplotlib>=3.9,<3.10", + "astropy>=6.1,<7", + "scikit-image>=0.24,<0.25", + "joblib>=1.4,<2", + "imageio>=2.34,<3", + "umap-learn>=0.5.6,<0.6", + "shap>=0.46.0,<0.47", + "slack-sdk>=3.31.0,<4", + "python-dotenv>=1.0,<2", + "huggingface_hub>=1.21,<1.22", + # setuptools provides pkg_resources, which blimpy (transitive via setigen) imports at + # module load; <81 because setuptools 81 removed the vendored pkg_resources. + "setuptools>=70,<81", + "setigen", + # The NGC base image provides these implicitly (so requirements-container.txt omits + # them); a pip install has no base image to lean on and must declare them. Ranges + # mirror environment.yml's pins. + "tensorflow[and-cuda]==2.17.*", + "h5py>=3.11,<3.12", + "hdf5plugin", + "psutil>=6.0,<6.1", ] license = "BSD-3-Clause" license-files = ["LICENSE"] @@ -35,12 +65,17 @@ keywords = [ "Data Methods", ] classifiers = [ - "Development Status :: 2 - Pre-Alpha", # TODO: update development status on v1 release + # TODO: v1 release PR — flip to "Development Status :: 4 - Beta" or + # "Development Status :: 5 - Production/Stable" (maintainer's call at release time; + # revisit in every release PR per docs/RELEASE.md) + "Development Status :: 2 - Pre-Alpha", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.12", "Operating System :: Unix", "Environment :: GPU :: NVIDIA CUDA :: 12 :: 12.4", - "Environment :: GPU :: NVIDIA CUDA :: 12 :: 12.8", + # NOTE: the NGC 25.02 container runs CUDA 12.8, but the trove classifier list currently + # tops out at "12 :: 12.6" — the bare ":: 12" parent is the closest valid classifier + "Environment :: GPU :: NVIDIA CUDA :: 12", "License :: OSI Approved :: BSD License", "Intended Audience :: Science/Research", "Topic :: Scientific/Engineering :: Artificial Intelligence", @@ -53,7 +88,7 @@ Homepage = "https://github.com/zachtheyek/Aetherscan" Issues = "https://github.com/zachtheyek/Aetherscan/issues" [project.optional-dependencies] -dev = ["ruff"] # Python linter +dev = ["ruff", "pre-commit", "pytest"] [tool.pytest.ini_options] # Make `import aetherscan` work without a PYTHONPATH=src prefix diff --git a/src/aetherscan/__init__.py b/src/aetherscan/__init__.py index 5f9989f..b2d4511 100644 --- a/src/aetherscan/__init__.py +++ b/src/aetherscan/__init__.py @@ -9,7 +9,16 @@ from __future__ import annotations -__version__ = "0.1.0" +from importlib.metadata import PackageNotFoundError, version + +# Version is single-sourced from pyproject.toml's [project].version via the installed +# distribution's metadata. Source-tree runs (PYTHONPATH=src, the NGC container) have no +# installed distribution — they fall back to a dev sentinel, which also keeps the +# version-coupled HF weight resolution (hf_hub.version_default_revision) inactive there. +try: + __version__ = version("aetherscan") +except PackageNotFoundError: + __version__ = "0.0.0.dev0" __author__ = "Zach Yek" # TODO: determine which components are necessary to expose for public API From 09508359b8b18560caaf16b3e149eac796dc7f72 Mon Sep 17 00:00:00 2001 From: zachtheyek Date: Sun, 12 Jul 2026 15:59:10 +0800 Subject: [PATCH 2/5] Version-couple HF weight resolution to the installed release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the inference revision-resolution chain with a version-pinned default between --hf-revision and latest-release resolution: explicit local paths > --hf-revision > v{__version__} > latest semver v* tag > latest final_vX tag > error with guidance. version_default_revision() makes 'pip install aetherscan==1.0.0' + bare inference pull exactly the v1.0.0-blessed weights — the runtime half of docs/RELEASE.md's contract. Guard: the default only activates when v{__version__} lands in the release-tag family (strict vX.Y.Z via _SEMVER_TAG_PATTERN). The importlib.metadata fallback (0.0.0.dev0), .dev pre-releases, and rc/post/local versions fall through to latest-release resolution, so source-tree and dev-install runs keep tracking the newest blessed weights. Deliberately not existence-checked: an installed release whose weights tag is missing fails the download loudly rather than silently pulling another version's weights. --hf-revision help text updated; README inference CLI block regenerated (top/train verified unchanged). Tests cover the guard shapes, precedence, pin-without-listing, dev fallthrough, and the end-to-end installed-release download; existing no-revision tests now pin the dev version explicitly so they never depend on whether aetherscan happens to be pip-installed. Co-Authored-By: Claude Fable 5 --- README.md | 7 ++-- src/aetherscan/cli.py | 2 +- src/aetherscan/hf_hub.py | 48 +++++++++++++++++++++--- tests/unit/test_hf_hub.py | 77 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 0ff89e3..f280ac1 100644 --- a/README.md +++ b/README.md @@ -899,9 +899,10 @@ options: --hf-revision HF_REVISION HuggingFace revision (tag, branch, or commit hash) to pin the model download to when no local artifact paths - are given (default: the repo's latest release tag — - highest semver vX.Y.Z tag, falling back to the highest - final_vX training tag) + are given (default: v{package version} when running as + an installed release, else the repo's latest release + tag — highest semver vX.Y.Z tag, falling back to the + highest final_vX training tag) --save-tag SAVE_TAG Tag for current pipeline run. Current timestamp used (YYYYMMDD_HHMMSS) if none specified --force-tag, --no-force-tag diff --git a/src/aetherscan/cli.py b/src/aetherscan/cli.py index 1751a6e..108155d 100644 --- a/src/aetherscan/cli.py +++ b/src/aetherscan/cli.py @@ -908,7 +908,7 @@ def _add_inference_flags_to(parser): "--hf-revision", type=str, default=None, - help="HuggingFace revision (tag, branch, or commit hash) to pin the model download to when no local artifact paths are given (default: the repo's latest release tag — highest semver vX.Y.Z tag, falling back to the highest final_vX training tag)", + help="HuggingFace revision (tag, branch, or commit hash) to pin the model download to when no local artifact paths are given (default: v{package version} when running as an installed release, else the repo's latest release tag — highest semver vX.Y.Z tag, falling back to the highest final_vX training tag)", ) # Checkpoint configuration diff --git a/src/aetherscan/hf_hub.py b/src/aetherscan/hf_hub.py index 238f6e2..34e9fb8 100644 --- a/src/aetherscan/hf_hub.py +++ b/src/aetherscan/hf_hub.py @@ -11,8 +11,10 @@ Upload (train, opt-in via --hf-upload) stages the four run artifacts under the stable names, commits them with the save_tag as the commit message, and tags the commit. Download (inference, the default when no local artifact paths are given) resolves a revision — -explicit --hf-revision, else the highest semver vX.Y.Z tag, else the highest final_vX tag — -and pulls the artifact trio revision-pinned via hf_hub_download. +explicit --hf-revision, else v{__version__} when running as an installed release (see +version_default_revision — this is what makes `pip install aetherscan==1.0.0` + bare +inference pull exactly the v1.0.0 weights), else the highest semver vX.Y.Z tag, else the +highest final_vX tag — and pulls the artifact trio revision-pinned via hf_hub_download. Auth: uploads require HF_TOKEN in the environment (loaded from the gitignored .env); huggingface_hub reads it implicitly. The repo is public, so downloads need no token. The @@ -129,15 +131,48 @@ def select_default_revision(tags: list[str]) -> str | None: return None +def version_default_revision() -> str | None: + """ + The version-coupled default HF revision — f"v{__version__}" — or None when this run's + version can't name a release tag. This is the runtime half of the release contract + (docs/RELEASE.md): `pip install aetherscan==1.0.0` + inference with no model-path flags + downloads exactly the v1.0.0-blessed weights, because the installed package version is + the default revision. + + Guard: the default only activates when v{__version__} lands in the release-tag family + (_SEMVER_TAG_PATTERN, strict vX.Y.Z). The importlib.metadata fallback ("0.0.0.dev0" — + source-tree/container runs), .dev pre-releases (e.g. "0.9.0.dev0" — a pip install of a + between-releases tree), and rc/post/local versions all fail the match and fall through + to latest-release resolution instead. Deliberately NOT existence-checked: an installed + release whose weights tag is missing must fail the download loudly (the release + runbook's blessing step was skipped) rather than silently pull some other version's + weights. + """ + import aetherscan # noqa: PLC0415 (late import so tests can monkeypatch __version__) + + candidate = f"v{aetherscan.__version__}" + if not _SEMVER_TAG_PATTERN.match(candidate): + return None + return candidate + + def resolve_hf_revision(repo_id: str, revision: str | None) -> str: """ Resolve the HF revision inference should download from: an explicitly requested revision - is returned as-is (existence is checked by the download itself), otherwise the repo's - tags are listed and select_default_revision picks the latest release. Raises RuntimeError - with guidance when nothing is resolvable. + is returned as-is (existence is checked by the download itself); otherwise an installed + release pins its own version's tag (version_default_revision — also download-checked); + otherwise the repo's tags are listed and select_default_revision picks the latest + release. Raises RuntimeError with guidance when nothing is resolvable. """ if revision is not None: return revision + version_pinned = version_default_revision() + if version_pinned is not None: + logger.info( + f"Resolved HF revision '{version_pinned}' (pinned to the installed aetherscan " + f"release; override with --hf-revision)" + ) + return version_pinned try: tags = list_hf_tags(repo_id) except Exception as e: @@ -188,7 +223,8 @@ def resolve_inference_artifacts(args: argparse.Namespace) -> None: """ Ensure the inference artifact trio (encoder/rf/config paths) is populated on `args` before validation runs, in resolution order: explicit local paths (highest) > - --hf-revision > latest release tag on the HF repo. + --hf-revision > v{__version__} when running as an installed release (see + version_default_revision) > latest release tag on the HF repo. When none of the three paths were given, the resolved revision's artifacts are downloaded and their cache paths written onto `args` — exactly as if the user had passed diff --git a/tests/unit/test_hf_hub.py b/tests/unit/test_hf_hub.py index 3126711..e875ccb 100644 --- a/tests/unit/test_hf_hub.py +++ b/tests/unit/test_hf_hub.py @@ -12,6 +12,7 @@ import numpy as np import pytest +import aetherscan from aetherscan import hf_hub from aetherscan.config import get_config from aetherscan.hf_hub import ( @@ -26,9 +27,17 @@ resolve_inference_artifacts, select_default_revision, upload_run_to_hf, + version_default_revision, ) +def _set_version(monkeypatch, value): + """Pin aetherscan.__version__ (read lazily by version_default_revision) for one test, so + resolution behavior never depends on whether the package happens to be pip-installed in + the environment running the suite.""" + monkeypatch.setattr(aetherscan, "__version__", value) + + class _FakeHfApi: """Records every HfApi call. upload_folder snapshots the staging directory's contents (the TemporaryDirectory is deleted right after the call, so assertions need a copy).""" @@ -118,6 +127,31 @@ def test_empty(self): assert select_default_revision([]) is None +class TestVersionDefaultRevision: + """The version-coupled default only activates for real installed releases (strict X.Y.Z); + every other version shape — crucially the source-tree fallback and .dev pre-releases — + must return None so resolution falls through to the repo's latest release tag.""" + + def test_installed_release_yields_version_tag(self, monkeypatch): + _set_version(monkeypatch, "1.2.3") + assert version_default_revision() == "v1.2.3" + + @pytest.mark.parametrize( + "version", + [ + "0.0.0.dev0", # the importlib.metadata PackageNotFoundError fallback + "0.9.0.dev0", # a pip install of a between-releases source tree + "1.0.0rc1", # release candidate + "1.0.0.post1", # post-release + "1.0.0+local", # local version + "1.0", # not a full X.Y.Z + ], + ) + def test_non_release_versions_are_guarded(self, monkeypatch, version): + _set_version(monkeypatch, version) + assert version_default_revision() is None + + class TestResolveHfRevision: def test_explicit_revision_returned_without_listing(self, monkeypatch): def boom(repo_id): @@ -126,11 +160,33 @@ def boom(repo_id): monkeypatch.setattr(hf_hub, "list_hf_tags", boom) assert resolve_hf_revision("ns/repo", "test_v17") == "test_v17" + def test_explicit_revision_outranks_installed_release(self, monkeypatch): + _set_version(monkeypatch, "1.2.3") + assert resolve_hf_revision("ns/repo", "final_v9") == "final_v9" + + def test_installed_release_pins_own_version_without_listing(self, monkeypatch): + # An installed release resolves v{__version__} directly — no tag listing, and no + # silent fallback to other weights (a missing tag must fail the download instead). + _set_version(monkeypatch, "1.2.3") + + def boom(repo_id): + raise AssertionError("must not list tags for a version-pinned revision") + + monkeypatch.setattr(hf_hub, "list_hf_tags", boom) + assert resolve_hf_revision("ns/repo", None) == "v1.2.3" + + def test_dev_version_falls_through_to_latest_release(self, monkeypatch): + _set_version(monkeypatch, "0.9.0.dev0") + monkeypatch.setattr(hf_hub, "list_hf_tags", lambda repo_id: ["final_v1", "v0.2.0"]) + assert resolve_hf_revision("ns/repo", None) == "v0.2.0" + def test_latest_release_selected(self, monkeypatch): + _set_version(monkeypatch, "0.0.0.dev0") monkeypatch.setattr(hf_hub, "list_hf_tags", lambda repo_id: ["final_v1", "v0.2.0"]) assert resolve_hf_revision("ns/repo", None) == "v0.2.0" def test_no_resolvable_tag_raises_with_guidance(self, monkeypatch): + _set_version(monkeypatch, "0.0.0.dev0") monkeypatch.setattr(hf_hub, "list_hf_tags", lambda repo_id: ["test_v1"]) with pytest.raises(RuntimeError, match="--hf-revision"): resolve_hf_revision("ns/repo", None) @@ -139,6 +195,7 @@ def test_listing_failure_wrapped_with_guidance(self, monkeypatch): def boom(repo_id): raise ConnectionError("no route to host") + _set_version(monkeypatch, "0.0.0.dev0") monkeypatch.setattr(hf_hub, "list_hf_tags", boom) with pytest.raises(RuntimeError, match="--hf-revision"): resolve_hf_revision("ns/repo", None) @@ -251,6 +308,7 @@ def fake_download(repo_id, filename, revision): assert args.config_path == f"/cache/{HF_CONFIG_FILENAME}" def test_no_paths_no_revision_resolves_latest_and_records_provenance(self, monkeypatch): + _set_version(monkeypatch, "0.0.0.dev0") monkeypatch.setattr(hf_hub, "list_hf_tags", lambda repo_id: ["v0.1.0", "final_v3"]) monkeypatch.setattr(hf_hub, "_hf_hub_download", lambda **kw: f"/cache/{kw['filename']}") args = self._args() @@ -259,6 +317,25 @@ def test_no_paths_no_revision_resolves_latest_and_records_provenance(self, monke # (and the saved inference config) via apply_args_to_config. assert args.hf_revision == "v0.1.0" + def test_installed_release_downloads_own_version_tag(self, monkeypatch): + # The release contract: pip install aetherscan==1.2.3 + bare inference downloads + # exactly the v1.2.3 weights, without ever listing the repo's tags. + _set_version(monkeypatch, "1.2.3") + monkeypatch.setattr( + hf_hub, "list_hf_tags", lambda repo_id: pytest.fail("must not list tags") + ) + downloads = [] + + def fake_download(**kw): + downloads.append(kw["revision"]) + return f"/cache/{kw['filename']}" + + monkeypatch.setattr(hf_hub, "_hf_hub_download", fake_download) + args = self._args() + resolve_inference_artifacts(args) + assert set(downloads) == {"v1.2.3"} + assert args.hf_revision == "v1.2.3" + def test_download_failure_wrapped_with_guidance(self, monkeypatch): def boom(**kwargs): raise ConnectionError("connection reset") From b8ad62c0a998af8989378a2601ae6e1068d9a86c Mon Sep 17 00:00:00 2001 From: zachtheyek Date: Sun, 12 Jul 2026 15:59:24 +0800 Subject: [PATCH 3/5] Add release CD: guard, tests, HF weights verify, build, PyPI, GH Release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .github/workflows/release.yml on push of v* tags, gates ordered so a half-released state is impossible: 1. Version guard — tag must equal v + pyproject version, else fail loudly (the release PR's bump didn't land on the tagged commit). 2. Unit tests — tests.yml called as a reusable workflow (it gains a workflow_call trigger), so the release gate runs the exact CI suite by construction rather than by copy-paste. 3. HF weights verification — the matching v* tag must exist on the public model repo (no token). Verify, never create: the failure message points at utils/hf_tag_release.py and docs/RELEASE.md step 3. 4. Build — python -m build (sdist+wheel) plus a wheel smoke: --no-deps install must report exactly the guarded version. 5. PyPI via trusted publishing — pypa/gh-action-pypi-publish, OIDC (id-token: write) scoped to the pypi environment; no stored token. One-time maintainer setup documented in the workflow header. 6. GitHub Release — gh release create --verify-tag --generate-notes with the dist files attached; body curated from claude-release-notes comments afterwards per the runbook. Dry run: workflow_dispatch with test_pypi publishes to test.pypi.org under the testpypi environment; manual runs refuse to target real PyPI. Co-Authored-By: Claude Fable 5 --- .github/workflows/release.yml | 233 ++++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 5 +- 2 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ee130ee --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,233 @@ +# Release CD — the automation half of the release contract in docs/RELEASE.md. +# +# Trigger: pushing a release tag (git tag -s vX.Y.Z && git push origin vX.Y.Z). The gates run +# in order — version guard -> unit tests -> HF weights verification -> build -> PyPI publish +# (trusted publishing) -> GitHub Release — so a half-released state is impossible: nothing +# publishes unless the tag matches pyproject.toml, the suite is green, and the blessed +# weights tag already exists on the HF model repo (CD verifies weights, never creates them — +# weights only come from cluster training runs). +# +# One-time maintainer prerequisites (see docs/RELEASE.md): +# - PyPI: create the `aetherscan` project with GitHub as a trusted publisher +# (repository zachtheyek/Aetherscan, workflow release.yml, environment pypi) and create +# the `pypi` environment in this repo's settings. No API token is stored anywhere. +# - For the TestPyPI dry run: the same trusted-publisher setup on test.pypi.org with +# environment `testpypi`. +name: Release + +on: + push: + tags: ["v*"] + # Dry-run path (recommended before the first real release): manual dispatch publishes to + # test.pypi.org. Real releases only ever happen via tag pushes — a manual run with + # test_pypi unchecked fails the guard. + workflow_dispatch: + inputs: + test_pypi: + description: "Publish to test.pypi.org (dry run)" + type: boolean + default: true + +permissions: + contents: read + +env: + # The HF model repo carrying the release weights. Must match config.hf.repo_id's default + # (src/aetherscan/config.py). + HF_WEIGHTS_REPO: zachtheyek/aetherscan + +jobs: + # Gate 1: fail loudly before anything runs if the pushed tag doesn't equal + # "v" + pyproject.toml's [project].version (i.e. the release PR's version bump didn't + # land on the tagged commit), or if a manual run tries to target the real PyPI. + guard: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.resolve.outputs.version }} + dry_run: ${{ steps.resolve.outputs.dry_run }} + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Resolve version and enforce tag match + id: resolve + env: + EVENT_NAME: ${{ github.event_name }} + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + TEST_PYPI: ${{ inputs.test_pypi }} + run: | + VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + if [ "$TEST_PYPI" != "true" ]; then + echo "::error::Manual runs only support the TestPyPI dry run (test_pypi: true)." \ + "Real releases must go through a signed tag push (docs/RELEASE.md step 5)." + exit 1 + fi + echo "dry_run=true" >> "$GITHUB_OUTPUT" + echo "Dry run: will publish aetherscan $VERSION to test.pypi.org" + exit 0 + fi + echo "dry_run=false" >> "$GITHUB_OUTPUT" + if [ "$REF_TYPE" != "tag" ]; then + echo "::error::Release runs must be triggered by a tag push, got ref type '$REF_TYPE'." + exit 1 + fi + if [ "$REF_NAME" != "v$VERSION" ]; then + echo "::error::Tag '$REF_NAME' does not match pyproject.toml version '$VERSION'" \ + "(expected tag 'v$VERSION'). Land the release PR that bumps the version," \ + "then tag its merge commit (docs/RELEASE.md steps 4-5)." + exit 1 + fi + echo "Version guard passed: $REF_NAME == v$VERSION" + + # Gate 2: a release build must not outrun a red suite. Calls tests.yml as a reusable + # workflow — the same suite and selection as regular CI, by construction. + tests: + needs: guard + uses: ./.github/workflows/tests.yml + + # Gate 3: verify the blessed weights tag exists on the HF model repo (public read, no + # token). Verify, never create: if this fails, the release runbook's weight-blessing step + # was skipped — the fix is utils/hf_tag_release.py, not anything in CI. + verify-weights: + needs: guard + runs-on: ubuntu-latest + steps: + - name: Skip on dry run + if: needs.guard.outputs.dry_run == 'true' + run: | + echo "Dry run: skipping HF weights verification — pre-release versions" \ + "(${{ needs.guard.outputs.version }}) never have a blessed weights tag." + + - name: Set up Python + if: needs.guard.outputs.dry_run != 'true' + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Verify HF weights tag exists + if: needs.guard.outputs.dry_run != 'true' + env: + RELEASE_TAG: v${{ needs.guard.outputs.version }} + run: | + # Range mirrors requirements-container.txt + pip install "huggingface_hub>=1.21,<1.22" + python - <<'EOF' + import os + import sys + + from huggingface_hub import HfApi + + repo_id = os.environ["HF_WEIGHTS_REPO"] + tag = os.environ["RELEASE_TAG"] + tags = [ref.name for ref in HfApi().list_repo_refs(repo_id, repo_type="model").tags] + if tag in tags: + print(f"Weights tag '{tag}' exists on https://huggingface.co/{repo_id} — OK") + sys.exit(0) + print( + f"::error::Weights tag '{tag}' not found on https://huggingface.co/{repo_id} " + f"(existing tags: {', '.join(sorted(tags)) or 'none'}). CD verifies release " + f"weights but never creates them. Bless the trained weights first — " + f"python utils/hf_tag_release.py --save-tag --release {tag} — " + f"then re-run this workflow. The git tag is already correct; do not re-push it. " + f"See docs/RELEASE.md step 3." + ) + sys.exit(1) + EOF + + # Build the sdist + wheel from the tagged source, and smoke the wheel: it must install + # and report exactly the guarded version. + build: + needs: [guard, tests, verify-weights] + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Build sdist and wheel + run: | + python -m pip install --upgrade pip build + python -m build + + - name: Smoke the wheel (installs and reports the guarded version) + env: + EXPECTED_VERSION: ${{ needs.guard.outputs.version }} + run: | + # --no-deps: the full dependency tree (TF + CUDA wheels) is exercised by the test + # jobs; here we only check packaging metadata and importability. + pip install --no-deps dist/*.whl + INSTALLED=$(python -c "import aetherscan; print(aetherscan.__version__)") + if [ "$INSTALLED" != "$EXPECTED_VERSION" ]; then + echo "::error::Installed wheel reports version '$INSTALLED', expected '$EXPECTED_VERSION'." + exit 1 + fi + echo "Wheel smoke passed: aetherscan.__version__ == $INSTALLED" + + - name: Upload distribution artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + # Publish via PyPI trusted publishing (OIDC): the `pypi` / `testpypi` GitHub environment + # plus id-token permission is the whole credential — no stored API token. + publish: + needs: [guard, build] + runs-on: ubuntu-latest + environment: + name: ${{ needs.guard.outputs.dry_run == 'true' && 'testpypi' || 'pypi' }} + url: ${{ needs.guard.outputs.dry_run == 'true' && 'https://test.pypi.org/p/aetherscan' || 'https://pypi.org/p/aetherscan' }} + permissions: + id-token: write + steps: + - name: Download distribution artifacts + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Publish to TestPyPI (dry run) + if: needs.guard.outputs.dry_run == 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + - name: Publish to PyPI + if: needs.guard.outputs.dry_run != 'true' + uses: pypa/gh-action-pypi-publish@release/v1 + + # Decorate the tag into a GitHub Release with generated notes and the built artifacts + # attached. The generated notes are a starting point — curate the body from the per-PR + # claude-release-notes comments afterwards (docs/RELEASE.md step 6). + github-release: + needs: [guard, publish] + if: needs.guard.outputs.dry_run != 'true' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Download distribution artifacts + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Create GitHub Release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: v${{ needs.guard.outputs.version }} + run: | + gh release create "$RELEASE_TAG" dist/* \ + --verify-tag \ + --title "$RELEASE_TAG" \ + --generate-notes diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5c6842d..50c3405 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,13 +1,16 @@ # Unit test suite (pytest) name: Tests -# Run on every PR and on pushes to master +# Run on every PR and on pushes to master. Also callable as a reusable workflow so the +# release CD (release.yml) runs the exact same suite as a publish gate — by construction, +# not by copy-paste. on: push: branches: ["master"] pull_request: branches: ["**"] workflow_dispatch: + workflow_call: permissions: contents: read From a77bfefc29049e729dacad108a3980c5e6863478 Mon Sep 17 00:00:00 2001 From: zachtheyek Date: Sun, 12 Jul 2026 15:59:37 +0800 Subject: [PATCH 4/5] Add utils/hf_tag_release.py: bless trained weights as a release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release runbook's step 3 (docs/RELEASE.md): create the HF release tag vX.Y.Z pointing at a training upload's commit — python utils/hf_tag_release.py --save-tag final_v1 --release v1.0.0 A thin wrapper over HfApi.create_tag, standalone on purpose (no PYTHONPATH or aetherscan install required). Validates the release-tag shape, loads HF_TOKEN from the gitignored .env when python-dotenv is available, and turns the common failure modes into targeted errors: missing token, save-tag not on the repo (train with --hf-upload first), and an already-existing release tag (immutable by convention). This is the human blessing decision the release CD deliberately cannot make — it only verifies the tag exists before publishing. Co-Authored-By: Claude Fable 5 --- utils/hf_tag_release.py | 101 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 utils/hf_tag_release.py diff --git a/utils/hf_tag_release.py b/utils/hf_tag_release.py new file mode 100644 index 0000000..494c8e9 --- /dev/null +++ b/utils/hf_tag_release.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +""" +Bless trained weights as a release: create the HF release tag (vX.Y.Z) pointing at a +training upload's commit (docs/RELEASE.md step 3). + + python utils/hf_tag_release.py --save-tag final_v1 --release v1.0.0 + +This is the human "these weights are the release" decision — the release CD workflow +(.github/workflows/release.yml) verifies the tag exists but deliberately cannot create it. +A thin wrapper over HfApi.create_tag: the new release tag points at the same commit as the +training run's save-tag (created by `train --hf-upload`). + +Auth: needs a write-scoped HF_TOKEN in the environment; a gitignored .env in the working +directory is loaded when python-dotenv is available. Standalone on purpose — no PYTHONPATH +or aetherscan install required. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys + +from huggingface_hub import HfApi +from huggingface_hub.errors import HfHubHTTPError, RevisionNotFoundError + +try: + from dotenv import load_dotenv +except ImportError: # python-dotenv is optional here — HF_TOKEN may already be exported + load_dotenv = None + +# Must match config.hf.repo_id's default (src/aetherscan/config.py). +DEFAULT_REPO_ID = "zachtheyek/aetherscan" + +_RELEASE_TAG_PATTERN = re.compile(r"^v\d+\.\d+\.\d+$") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Bless trained weights as a release: create the HF release tag (vX.Y.Z) " + "pointing at a training upload's commit (docs/RELEASE.md step 3)." + ) + parser.add_argument( + "--save-tag", + required=True, + help="Training run tag whose uploaded weights are being blessed (e.g. final_v1); " + "must already exist on the HF repo (created by `train --hf-upload`)", + ) + parser.add_argument( + "--release", + required=True, + help="Release tag to create, vX.Y.Z (must match the version the release PR ships)", + ) + parser.add_argument( + "--repo-id", + default=DEFAULT_REPO_ID, + help=f"HF model repo id (default: {DEFAULT_REPO_ID})", + ) + args = parser.parse_args() + + if not _RELEASE_TAG_PATTERN.match(args.release): + print(f"ERROR: --release must look like vX.Y.Z (e.g. v1.0.0), got {args.release!r}") + return 1 + + if load_dotenv is not None: + load_dotenv() + if not os.environ.get("HF_TOKEN"): + print( + "ERROR: no HF_TOKEN in the environment. Creating tags needs a write-scoped " + "token — put it in the gitignored .env (never commit it)." + ) + return 1 + + api = HfApi() + try: + api.create_tag(args.repo_id, tag=args.release, revision=args.save_tag) + except RevisionNotFoundError: + print( + f"ERROR: save-tag {args.save_tag!r} not found on {args.repo_id}. Was the run " + f"trained with --hf-upload (and did the hf_upload stage succeed)?" + ) + return 1 + except HfHubHTTPError as e: + if getattr(getattr(e, "response", None), "status_code", None) == 409: + print( + f"ERROR: release tag {args.release!r} already exists on {args.repo_id}. " + f"Released weights are immutable by convention — bless a new version " + f"instead, or delete the tag on the Hub first if it was created in error." + ) + return 1 + raise + print( + f"Blessed: {args.release} -> weights of training run {args.save_tag!r} " + f"(https://huggingface.co/{args.repo_id}/tree/{args.release})" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 57089c8db836b96e1d8afed919830d9a9310d3f6 Mon Sep 17 00:00:00 2001 From: zachtheyek Date: Sun, 12 Jul 2026 16:08:43 +0800 Subject: [PATCH 5/5] Apply review: close pyproject handle in guard, validate --save-tag, pin setigen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - release.yml version guard reads pyproject.toml via a with statement instead of leaving the handle to the GC (reviewer nit). - hf_tag_release.py rejects an empty/whitespace --save-tag up front with a clear message instead of surfacing it as a Hub 404 later. - setigen gains a >=2.6,<3 range (proven 2.6/2.7 line, 2.7.0 in current use) in pyproject.toml, requirements-container.txt, and environment.yml in lockstep — it was the only unpinned dependency. Co-Authored-By: Claude Fable 5 --- .github/workflows/release.yml | 6 +++++- environment.yml | 3 ++- pyproject.toml | 2 +- requirements-container.txt | 4 +++- utils/hf_tag_release.py | 6 ++++++ 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee130ee..890890a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,7 +57,11 @@ jobs: REF_NAME: ${{ github.ref_name }} TEST_PYPI: ${{ inputs.test_pypi }} run: | - VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") + VERSION=$(python3 -c " + import tomllib + with open('pyproject.toml', 'rb') as f: + print(tomllib.load(f)['project']['version']) + ") echo "version=$VERSION" >> "$GITHUB_OUTPUT" if [ "$EVENT_NAME" = "workflow_dispatch" ]; then if [ "$TEST_PYPI" != "true" ]; then diff --git a/environment.yml b/environment.yml index da45184..4d5e6e9 100644 --- a/environment.yml +++ b/environment.yml @@ -51,7 +51,8 @@ dependencies: # nvidia-nccl-cu12 2.21.x, etc. -- the exact ABI the TF 2.17 wheel was built # against. Driver requirement: NVIDIA driver >= 525.60 on the host. - tensorflow[and-cuda]==2.17.* - - setigen + # Range mirrors requirements-container.txt (see the rationale there). + - setigen>=2.6,<3 - imageio>=2.34,<3 - umap-learn>=0.5.6,<0.6 - shap>=0.46.0,<0.47 diff --git a/pyproject.toml b/pyproject.toml index fabb8cb..3ee9b5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ # setuptools provides pkg_resources, which blimpy (transitive via setigen) imports at # module load; <81 because setuptools 81 removed the vendored pkg_resources. "setuptools>=70,<81", - "setigen", + "setigen>=2.6,<3", # The NGC base image provides these implicitly (so requirements-container.txt omits # them); a pip install has no base image to lean on and must declare them. Ranges # mirror environment.yml's pins. diff --git a/requirements-container.txt b/requirements-container.txt index cee242f..65063d4 100644 --- a/requirements-container.txt +++ b/requirements-container.txt @@ -30,4 +30,6 @@ huggingface_hub>=1.21,<1.22 # setuptools 81 removed pkg_resources as a vendored submodule -- revisit # once blimpy migrates off pkg_resources (upstream issue). setuptools>=70,<81 -setigen +# Range covers the proven 2.6/2.7 line (2.7.0 in current use) and blocks a future +# major-version break; keep in lockstep with environment.yml and pyproject.toml. +setigen>=2.6,<3 diff --git a/utils/hf_tag_release.py b/utils/hf_tag_release.py index 494c8e9..443b03d 100644 --- a/utils/hf_tag_release.py +++ b/utils/hf_tag_release.py @@ -62,6 +62,12 @@ def main() -> int: if not _RELEASE_TAG_PATTERN.match(args.release): print(f"ERROR: --release must look like vX.Y.Z (e.g. v1.0.0), got {args.release!r}") return 1 + # Fail a mangled save-tag here with a clear message rather than as a Hub 404 later. + if not args.save_tag or any(c.isspace() for c in args.save_tag): + print( + f"ERROR: --save-tag must be a non-empty tag without whitespace, got {args.save_tag!r}" + ) + return 1 if load_dotenv is not None: load_dotenv()