build: ship nnnotes as a pure Python package with the deck model as the extra deck

.github/scripts/smoke_wheel.py

@@ -1,4 +1,4 @@
-"""Install one wheel in a fresh environment and exercise its CLI and extension."""
+"""Install the wheel with the deck model in a fresh environment and exercise its CLI and the deck model."""
 from pathlib import Path
 import os
 import subprocess
@@ -13,20 +13,21 @@
 import sys
 
 import nnnotes
-from nnnotes import _deck, deckdata
+import ournotes_sim
+from nnnotes import deckdata, musicdata
 
 assert Path(nnnotes.__file__).resolve().is_relative_to(Path(sys.prefix).resolve())
-assert Path(_deck.__file__).resolve().is_relative_to(Path(sys.prefix).resolve())
-info = _deck.info()
+deck = musicdata.Deck(workers=1, aptitude=False)
+info = deck.info()
 data = {
-    "format": info["dataFormat"],
+    "format": ournotes_sim.info()["dataFormat"],
     "master": {name: {"columns": [], "rows": []} for name, _ in deckdata.TABLES},
     "charts": [],
 }
-stats = json.loads(_deck.chart_stats(json.dumps(data), workers=1, aptitude=False))
+stats = json.loads(ournotes_sim.chart_stats(json.dumps(data), workers=1, aptitude=False))
 assert stats["format"] == info["format"]
 assert stats["charts"] == []
-print(f"nnnotes {nnnotes.__version__}: {info['name']} {info['version']}")
+print(f"nnnotes {nnnotes.__version__}: {info['name']} {info['version']} ({info['commit']})")
 """
 
 
@@ -38,7 +39,7 @@ def main(directory: str) -> None:
         venv.EnvBuilder(with_pip=True).create(environment)
         scripts = environment / ("Scripts" if os.name == "nt" else "bin")
         python = scripts / ("python.exe" if os.name == "nt" else "python")
-        subprocess.run([python, "-I", "-m", "pip", "install", str(wheel)], cwd=root, check=True)
+        subprocess.run([python, "-I", "-m", "pip", "install", f"{wheel}[deck]"], cwd=root, check=True)
         subprocess.run([python, "-I", "-c", SMOKE], cwd=root, check=True)
         subprocess.run([scripts / ("nnnotes.exe" if os.name == "nt" else "nnnotes"), "--help"],
                        cwd=root, check=True)

.github/workflows/ci.yml

@@ -29,13 +29,10 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: pip
           cache-dependency-path: pyproject.toml
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Install (builds the extension module nnnotes._deck)
+      - name: Install (with the deck model)
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -e ".[test]"
+          python -m pip install -e ".[test,deck]"
       - name: Lint
         run: python -m pyflakes src tests
       - name: Command line
@@ -56,9 +53,6 @@ jobs:
           cache-dependency-path: |
             pyproject.toml
             .github/goldens-constraints.txt
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
       - name: Install (pinned)
         run: |
           python -m pip install --upgrade pip
@@ -67,17 +61,3 @@ jobs:
         env:
           GOLDENS_STRICT: "1"
         run: python -m pytest tests/test_goldens.py tests/test_objexport.py
-
-  rust:
-    # the extension module's own code (rust/); ournotes-deck is tested in its repository
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v7
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: rust
-      - name: Format and lints
-        working-directory: rust
-        run: |
-          cargo fmt --check
-          cargo clippy --locked --all-targets -- -D warnings

.github/workflows/deck.yml

@@ -1,11 +1,11 @@
 name: Deck model
 
-# nnnotes builds the deck model, the crate ournotes-sim of the ournotes-deck repository, at the commit rust/Cargo.toml
-# pins, so that one nnnotes version always carries one deck model. The pin follows ournotes-deck's releases: when the
-# latest release has published its replay and recommendation WASM packages (the engines of music-data --replay-engine
-# and --recommend-engine) and its commit is past the pinned one, this opens a pull request that moves the pin and
-# dispatches CI and the commit message check on its branch (a pull request opened with the workflow token starts no
-# workflow by itself); their checks land on the pull request.
+# The extra `deck` pins the deck model, the package ournotes-sim that ournotes-deck publishes with each release, so
+# that one nnnotes version always carries one deck model. The pin follows ournotes-deck's releases: when the latest
+# release has published its replay and recommendation WASM packages (the engines of music-data --replay-engine and
+# --recommend-engine) and its ournotes-sim wheels, and it is past the pinned release, this opens a pull request that
+# moves the pin and dispatches CI and the commit message check on its branch (a pull request opened with the workflow
+# token starts no workflow by itself); their checks land on the pull request.
 on:
   schedule:
     - cron: "17 3 * * *"
@@ -21,35 +21,32 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - name: Compare the pinned commit with ournotes-deck's latest release
+      - name: Compare the pinned release with ournotes-deck's latest release
         id: deck
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          pinned=$(sed -nE 's/^ournotes-sim = .*rev = "([0-9a-f]{40})".*/\1/p' rust/Cargo.toml)
-          [ -n "$pinned" ] || { echo "rust/Cargo.toml pins no ournotes-sim commit" >&2; exit 1; }
+          pinned=$(sed -nE 's/^deck = \["ournotes-sim==([0-9]+\.[0-9]+\.[0-9]+)"\]$/\1/p' pyproject.toml)
+          [ -n "$pinned" ] || { echo "pyproject.toml pins no ournotes-sim release" >&2; exit 1; }
           release=$(gh api repos/empty-sekai/ournotes-deck/releases/latest)
           tag=$(jq -r .tag_name <<< "$release")
-          head=$(gh api "repos/empty-sekai/ournotes-deck/commits/$tag" --jq .sha)
+          version=${tag#v}
           published=true
           for module in replay recommend; do
             jq -e --arg name "ournotes-$module-wasm-$tag.tar.gz" 'any(.assets[]; .name == $name)' <<< "$release" \
               > /dev/null || published=false
           done
-          # ahead: the release contains the pinned commit and more; behind or diverged: leave the pin to a person
-          # (with no common history the comparison itself fails)
-          status=$(gh api "repos/empty-sekai/ournotes-deck/compare/$pinned...$head" --jq .status)
-          echo "pinned $pinned, $tag $head: $status, WASM packages published: $published"
+          curl -fsS -o /dev/null "https://pypi.org/pypi/ournotes-sim/$version/json" || published=false
+          # ahead: the release contains the pinned release and more; behind or diverged: leave the pin to a person
+          status=$(gh api "repos/empty-sekai/ournotes-deck/compare/v$pinned...$tag" --jq .status)
+          echo "pinned $pinned, latest $version: $status, WASM packages and wheels published: $published"
           echo "pinned=$pinned" >> "$GITHUB_OUTPUT"
-          echo "head=$head" >> "$GITHUB_OUTPUT"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
           echo "move=$([ "$status" = ahead ] && [ "$published" = true ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
       - name: Move the pin
         if: steps.deck.outputs.move == 'true'
-        working-directory: rust
-        run: |
-          sed -i -E 's/(rev = ")${{ steps.deck.outputs.pinned }}(")/\1${{ steps.deck.outputs.head }}\2/' Cargo.toml
-          cargo update -p ournotes-sim
+        run: sed -i -E 's/^(deck = \["ournotes-sim==)${{ steps.deck.outputs.pinned }}("\])$/\1${{ steps.deck.outputs.version }}\2/' pyproject.toml
       - uses: peter-evans/create-pull-request@v8
         id: pr
         if: steps.deck.outputs.move == 'true'
@@ -58,9 +55,9 @@ jobs:
           commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}"
           title: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}"
           body: |
-            Moves the deck model to ournotes-deck ${{ steps.deck.outputs.tag }} (${{ steps.deck.outputs.head }}),
+            Moves the deck model to ournotes-deck ${{ steps.deck.outputs.tag }},
             https://github.com/empty-sekai/ournotes-deck/releases/tag/${{ steps.deck.outputs.tag }}
-            Changes: https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }}
+            Changes: https://github.com/empty-sekai/ournotes-deck/compare/v${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.tag }}
 
             Music data files made after this change name the new commit in `provenance.deck.commit`; their deck
             statistics may differ.

.github/workflows/release.yml

@@ -13,13 +13,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - name: Check the tag against the package and extension versions
+      - name: Check the tag against the package version
         if: startsWith(github.ref, 'refs/tags/v')
         run: |
           version=$(python3 -c 'import re, pathlib; print(re.search(r"__version__ = \"([^\"]+)\"", pathlib.Path("src/nnnotes/__init__.py").read_text()).group(1))')
-          cargo=$(python3 -c 'import re, pathlib; print(re.search(r"^version = \"([^\"]+)\"", pathlib.Path("rust/Cargo.toml").read_text(), re.M).group(1))')
-          if [ "v$version" != "$GITHUB_REF_NAME" ] || [ "$cargo" != "$version" ]; then
-            echo "tag $GITHUB_REF_NAME, package $version and rust/Cargo.toml $cargo differ" >&2
+          if [ "v$version" != "$GITHUB_REF_NAME" ]; then
+            echo "tag $GITHUB_REF_NAME and package $version differ" >&2
             exit 1
           fi
           if ! grep -qF "## [$version]" CHANGELOG.md; then
@@ -31,68 +30,23 @@ jobs:
     needs: check
     uses: ./.github/workflows/ci.yml
 
-  # one abi3 wheel (CPython 3.11 and later) per platform with the extension module nnnotes._deck, and the sdist
-  wheels:
-    needs: check
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - {os: ubuntu-latest, target: x86_64, manylinux: "2_28"}
-          - {os: ubuntu-24.04-arm, target: aarch64, manylinux: "2_28"}
-          - {os: windows-latest, target: x64}
-          - {os: macos-latest, target: aarch64}
-          - {os: macos-15-intel, target: x86_64}
-    steps:
-      - uses: actions/checkout@v7
-      - uses: actions/setup-python@v7
-        with:
-          python-version: "3.13"
-      - uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.target }}
-          manylinux: ${{ matrix.manylinux || 'auto' }}
-          args: --release --locked --out dist
-      - name: Test the installed wheel
-        run: python .github/scripts/smoke_wheel.py dist
-      - uses: actions/upload-artifact@v7
-        with:
-          name: wheel-${{ matrix.os }}-${{ matrix.target }}
-          path: dist/
-          if-no-files-found: error
-
-  sdist:
-    needs: check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v7
-      - uses: PyO3/maturin-action@v1
-        with:
-          command: sdist
-          args: --out dist
-      - uses: actions/upload-artifact@v7
-        with:
-          name: sdist
-          path: dist/
-          if-no-files-found: error
-
+  # the sdist and one pure Python wheel; the wheel is tested with the deck model the extra pins
   build:
-    needs: [quality, wheels, sdist]
+    needs: [check, quality]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/download-artifact@v8
-        with:
-          path: dist/
-          merge-multiple: true
+      - uses: actions/checkout@v7
       - uses: actions/setup-python@v7
         with:
           python-version: "3.13"
-      - name: Check the distributions
+      - name: Build and check the distributions
         run: |
-          python -m pip install --upgrade pip twine
+          python -m pip install --upgrade pip build twine
+          python -m build --outdir dist
           python -m twine check --strict dist/*
           ls dist
+      - name: Test the installed wheel
+        run: python .github/scripts/smoke_wheel.py dist
       - uses: actions/upload-artifact@v7
         with:
           name: dist

.gitignore

@@ -9,6 +9,3 @@ nnnotes.toml
 .env
 out/
 cache/
-rust/target/
-src/nnnotes/_deck*.pyd
-src/nnnotes/_deck*.so

README.en.md

@@ -28,7 +28,8 @@ nnnotes config check             # each setting's origin and whether it is valid
 - Scripts and AI agents get no questions: `nnnotes config init --user --no-input --set SECTION.KEY=VALUE ...` writes
   the file at once, `nnnotes config set SECTION.KEY VALUE` changes one value (VALUE `-` reads it from standard input,
   so a key need not appear on the command line), and `nnnotes config check --json` reports the states for machines.
-- `--fonts game` also needs `pip install 'nnnotes[fonts]'`; the APK, the master data and the external tools are listed
+- `--fonts game` also needs `pip install 'nnnotes[fonts]'`, and the deck statistics of `music-data` need
+  `pip install 'nnnotes[deck]'`; the APK, the master data and the external tools are listed
   under Requirements below.
 
 ## Features
@@ -54,7 +55,7 @@ nnnotes config check             # each setting's origin and whether it is valid
 | `ui` (experimental) | APK set and its embedded catalog | an offline serialized prefab library with dependency roots, AnimatorControllers, stable references and PNG/font resources for the optional ournotes-player UI preview ([format and limits](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/ui.md)) |
 | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing |
 | `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) |
-| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; the Gekisou catalog (member cards, snaps and their Gekisou skills and support skills); per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position), plus single-shape Gekisou aptitude (increments and standard errors, not an optimal deck; omit with `--no-gekisou-aptitude`). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) |
+| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; the Gekisou catalog (member cards, snaps and their Gekisou skills and support skills); per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (installed with `pip install 'nnnotes[deck]'`) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position), plus single-shape Gekisou aptitude (increments and standard errors, not an optimal deck; omit with `--no-gekisou-aptitude`). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) |
 
 Export conventions: