Do not publish an unchanged index

.github/workflows/snapshot.yml

@@ -24,13 +24,17 @@ concurrency:
   group: snapshot
   cancel-in-progress: false
 
+env:
+  PUBLISHED: https://ksamodding.github.io/content-index-releases/v1/index.json
+
 jobs:
   build:
     name: build
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    outputs:
+      changed: ${{ steps.compare.outputs.changed }}
     steps:
-      # Both halves are public and this job only reads, so no App token.
       - uses: actions/checkout@v7
         with:
           ref: main
@@ -41,24 +45,44 @@ jobs:
           ref: main
           path: .authored
 
+      - name: Fetch the published snapshot
+        run: |
+          set -euo pipefail
+          curl -fsS --max-time 30 -H 'Cache-Control: no-cache' \
+            "$PUBLISHED" -o published.json || rm -f published.json
+
       - name: Build
         run: |
           set -euo pipefail
           python3 tools/build_snapshot.py \
             --authored .authored \
+            --previous published.json \
             --out _site/v1/index.json \
             --authored-repo "${{ github.repository_owner }}/content-index" \
             --authored-commit "$(git -C .authored rev-parse HEAD)" \
             --generated-repo "${{ github.repository }}" \
             --generated-commit "$(git rev-parse HEAD)"
 
-      - uses: actions/upload-pages-artifact@v5
+      - name: Compare against what is published
+        id: compare
+        run: |
+          set -euo pipefail
+          if [ -f published.json ] && cmp -s published.json _site/v1/index.json; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "the published snapshot is already current, so nothing is deployed"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - if: steps.compare.outputs.changed == 'true'
+        uses: actions/upload-pages-artifact@v5
         with:
           path: _site
 
   deploy:
     name: deploy
     needs: build
+    if: needs.build.outputs.changed == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 10
     # Pages serves the strong ETag and answers If-None-Match

README.md

@@ -40,7 +40,9 @@ That poll only ever sees the build that is current when it runs, so a build supe
 ## The watcher
 
 `.github/workflows/watcher.yml` runs every ten minutes as the org App.
-Each tick asks every listing's authority host for its releases and stamps every release that has no file under `releases/<id>/` yet, so a release published out of version order is stamped too.
+Each tick asks every listing's authority host for its releases and stamps every release that appeared after the newest one already stamped, so a patch for an older line, tagged after a newer version exists, is stamped too.
+
+A listing's first tick stamps its newest release only, and its back catalogue stays unstamped.
 
 There is no queue. What is stamped here is the whole of the watcher's state, which is why a tick GitHub delays, drops or cancels costs latency and not data, and why a re-run stamps nothing twice.
 
@@ -82,6 +84,14 @@ python3 tools/build_snapshot.py --authored ../content-index --out _site/v1/index
 
 The `sources` block naming the two commits is omitted unless both repositories and both commits are given, so a local build does not claim a provenance it does not have.
 
+### An unchanged index is not published again
+
+A deployment issues a new ETag whether or not the bytes changed, so republishing an unchanged snapshot is a full re-download for every client.
+
+The build reads what is published today as `--previous`. Unchanged content keeps that copy's `sources` rather than restamping whatever HEAD the run saw, so an unrelated commit does not move the bytes, and the workflow then skips the deploy when the two are equal.
+
+A build fails closed: nothing is published, and clients keep the last good snapshot.
+
 ## A published release is immutable
 
 Identity, the version, the download and the install data never change.

tools/build_snapshot.py

@@ -394,6 +394,61 @@ def serialize(document):
     return json.dumps(document, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
 
 
+def body(document):
+    """The document without its provenance, which is what "unchanged" means."""
+    return {key: value for key, value in document.items() if key != "sources"}
+
+
+def with_sources(document, sources):
+    """`document` with its provenance replaced, in the field order of the format."""
+    rebuilt = {"snapshot_version": document["snapshot_version"]}
+    if sources is not None:
+        rebuilt["sources"] = sources
+    for key, value in document.items():
+        if key not in ("snapshot_version", "sources"):
+            rebuilt[key] = value
+    return rebuilt
+
+
+def read_previous(path, log=warn):
+    """The snapshot published today, or None. Never raises: it comes over the network.
+
+    `NaN` and `Infinity` are refused for the reason load_json refuses them.
+    """
+    if path is None:
+        return None
+
+    def reject(literal):
+        raise ValueError(f"{literal} is not valid JSON")
+
+    try:
+        document = json.loads(Path(path).read_text(encoding="utf-8"), parse_constant=reject)
+    except FileNotFoundError:
+        return None
+    except (OSError, ValueError) as error:
+        log(f"{path} could not be read, so the provenance is not carried forward: {error}")
+        return None
+    if not isinstance(document, dict):
+        log(f"{path} is not an object, so the provenance is not carried forward")
+        return None
+    return document
+
+
+def carry_forward(document, previous, log=info):
+    """Keep the published provenance while the content is unchanged.
+
+    Otherwise an unrelated commit moves the bytes and every client re-downloads.
+    The test is the bytes, because that is what the publish step compares.
+    """
+    if previous is None:
+        return document
+    candidate = with_sources(document, previous.get("sources"))
+    if serialize(candidate) != serialize(previous):
+        return document
+    log("the content is unchanged, so the published provenance is kept and the bytes stay identical")
+    return candidate
+
+
 def parse_arguments(argv):
     parser = argparse.ArgumentParser(
         description="Merge both halves of the index into one snapshot document."
@@ -405,6 +460,11 @@ def parse_arguments(argv):
     )
     parser.add_argument("--releases", default="releases", type=Path)
     parser.add_argument("--game-versions", default="game-versions.json", type=Path)
+    parser.add_argument(
+        "--previous", type=Path,
+        help="the snapshot published today. Unchanged content keeps its provenance, "
+        "so the bytes stay identical and the deploy can be skipped",
+    )
     parser.add_argument(
         "--out", type=Path,
         help="write here instead of to stdout. Never inside this repository: the "
@@ -459,6 +519,7 @@ def main(argv=None):
             sources=sources_from(arguments),
             log=warn,
         )
+        document = carry_forward(document, read_previous(arguments.previous))
         rendered = serialize(document)
     except SnapshotError as error:
         print(f"cannot build the snapshot: {error}", file=sys.stderr)

tools/test_build_snapshot.py

@@ -15,11 +15,15 @@
 
 from build_snapshot import (
     SnapshotError,
+    body,
     build,
+    carry_forward,
     precedence,
+    read_previous,
     serialize,
     sources_from,
     warn,
+    with_sources,
 )
 
 GAME_VERSIONS = {
@@ -598,6 +602,134 @@ def test_a_commit_that_came_back_empty_is_an_error(self):
             )
 
 
+SOURCES = {
+    "authored": {"repository": "KSAModding/content-index", "commit": "9fe1c0f"},
+    "generated": {"repository": "KSAModding/content-index-releases", "commit": "3a77b21"},
+}
+
+LATER = {
+    "authored": {"repository": "KSAModding/content-index", "commit": "aaaaaaa"},
+    "generated": {"repository": "KSAModding/content-index-releases", "commit": "bbbbbbb"},
+}
+
+
+class CarryForward(Fixture):
+    """`sources` must not turn an unrelated commit into a re-download for everyone."""
+
+    def published(self, sources=SOURCES):
+        """A snapshot of the index as it stands, as the published copy would be."""
+        return json.loads(serialize(self.index.build(sources=sources)))
+
+    def test_an_unchanged_index_keeps_the_published_provenance(self):
+        self.index.listing("AutoStage")
+        previous = self.published()
+        built = self.index.build(sources=LATER)
+        self.assertEqual(carry_forward(built, previous, self.index.notes.append)["sources"], SOURCES)
+
+    def test_an_unchanged_index_keeps_identical_bytes(self):
+        self.index.listing("AutoStage")
+        self.index.release("AutoStage", "0.4.3")
+        previous = self.published()
+        built = carry_forward(self.index.build(sources=LATER), previous, self.index.notes.append)
+        self.assertEqual(serialize(built), serialize(previous))
+
+    def test_changed_content_takes_the_new_provenance(self):
+        self.index.listing("AutoStage")
+        previous = self.published()
+        self.index.listing("DeltaVMap")
+        built = carry_forward(self.index.build(sources=LATER), previous, self.index.notes.append)
+        self.assertEqual(built["sources"], LATER)
+
+    def test_a_published_copy_without_a_provenance_is_reproduced_exactly(self):
+        self.index.listing("AutoStage")
+        previous = self.published(sources=None)
+        built = carry_forward(self.index.build(sources=LATER), previous, self.index.notes.append)
+        self.assertNotIn("sources", built)
+        self.assertEqual(serialize(built), serialize(previous))
+
+    def test_the_carried_document_keeps_the_field_order_of_the_format(self):
+        self.index.listing("AutoStage")
+        previous = self.published()
+        built = carry_forward(self.index.build(sources=LATER), previous, self.index.notes.append)
+        self.assertEqual(
+            list(built), ["snapshot_version", "sources", "listings", "packs", "game_versions"]
+        )
+
+    def test_no_published_copy_leaves_the_document_alone(self):
+        self.index.listing("AutoStage")
+        built = self.index.build(sources=LATER)
+        self.assertIs(carry_forward(built, None, self.index.notes.append), built)
+
+    def test_a_key_order_change_inside_a_listing_is_a_content_change(self):
+        """Parsed equality would call this unchanged, and the bytes are not."""
+        self.index.listing("AutoStage")
+        previous = self.published()
+        authored = previous["listings"][0]["authored"]
+        previous["listings"][0]["authored"] = {key: authored[key] for key in reversed(list(authored))}
+        built = carry_forward(self.index.build(sources=LATER), previous, self.index.notes.append)
+        self.assertEqual(built["sources"], LATER)
+
+    def test_the_published_copy_read_back_from_its_own_bytes_is_unchanged(self):
+        self.index.listing("AutoStage")
+        self.index.release("AutoStage", "0.4.3")
+        self.index.pack("Pack", "1.0.0")
+        built = self.index.build(sources=SOURCES)
+        previous = json.loads(serialize(built))
+        carried = carry_forward(built, previous, self.index.notes.append)
+        self.assertEqual(serialize(carried), serialize(previous))
+
+    def test_body_leaves_out_the_provenance_and_nothing_else(self):
+        document = self.index.build(sources=SOURCES)
+        self.assertNotIn("sources", body(document))
+        self.assertEqual(sorted(body(document)), ["game_versions", "listings", "packs", "snapshot_version"])
+
+    def test_with_sources_can_remove_the_block(self):
+        document = self.index.build(sources=SOURCES)
+        self.assertNotIn("sources", with_sources(document, None))
+
+
+class Previous(unittest.TestCase):
+    """Reading the published copy is a network fetch, so nothing here may raise."""
+
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.directory.cleanup)
+        self.root = Path(self.directory.name)
+        self.notes = []
+
+    def write(self, text):
+        path = self.root / "published.json"
+        path.write_text(text, encoding="utf-8")
+        return path
+
+    def test_no_path_at_all_reads_as_nothing_published(self):
+        self.assertIsNone(read_previous(None, self.notes.append))
+        self.assertEqual(self.notes, [])
+
+    def test_a_missing_file_reads_as_nothing_published_and_is_not_worth_a_note(self):
+        self.assertIsNone(read_previous(self.root / "absent.json", self.notes.append))
+        self.assertEqual(self.notes, [])
+
+    def test_a_body_that_does_not_parse_is_a_note_and_not_an_error(self):
+        self.assertIsNone(read_previous(self.write("{oops"), self.notes.append))
+        self.assertEqual(len(self.notes), 1)
+
+    def test_a_document_that_is_not_an_object_is_a_note_and_not_an_error(self):
+        self.assertIsNone(read_previous(self.write("[]"), self.notes.append))
+        self.assertEqual(len(self.notes), 1)
+
+    def test_a_nan_literal_is_a_note_and_not_an_error(self):
+        # Python reads it, JSON has no such literal, and serialize refuses it.
+        self.assertIsNone(read_previous(self.write('{"snapshot_version": NaN}'), self.notes.append))
+        self.assertEqual(len(self.notes), 1)
+
+    def test_a_published_snapshot_reads_back(self):
+        self.assertEqual(
+            read_previous(self.write('{"snapshot_version": 1}'), self.notes.append),
+            {"snapshot_version": 1},
+        )
+
+
 class Notes(unittest.TestCase):
     def run_warn(self, actions):
         environment = dict(os.environ)