Keep builds and uploads working across compiler and metadata updates

.github/workflows/pre-commit.yml

@@ -20,3 +20,6 @@ jobs:
 
       - name: Run pre-commit
         run: pixi run lint
+
+      - name: Test compatibility commits
+        run: pixi run python -m unittest discover -s tests -v

scripts/build-all.py

@@ -75,6 +75,10 @@ def main() -> None:
 
         command.extend(
             [
+                # A shared output channel can hide compatible published builds
+                # behind another recipe's newer, incompatible compiler pin.
+                "--output-dir",
+                str(Path("output") / recipe_dir.name),
                 "--variant-config",
                 variant_config,
                 "--skip-existing=all",

scripts/common.py

@@ -54,6 +54,15 @@ def commit_push_changes(message: str, branch_name: str) -> None:
 
     # Commit, pull and push the changes
     run_command(["git", "pull", "origin", branch_name])
+    # Another runner may already have pushed the same generated changes.
+    # Pulling that commit leaves nothing staged, which is a successful no-op.
+    result = run_command_unchecked(["git", "diff-index", "--cached", "--quiet", "HEAD"])
+    if result.returncode == 0:
+        eprint("No changes to commit after pulling.")
+        return
+    if result.returncode != 1:
+        sys.exit(result.returncode)
+
     run_command(["git", "commit", "--message", message, "--no-verify"])
     run_command(["git", "push", "--set-upstream", "origin", branch_name])
 

tests/test_build_all.py

@@ -0,0 +1,136 @@
+"""Exercise build-all dependency resolution with local package channels."""
+
+from contextlib import chdir
+import importlib
+import io
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tarfile
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+build_all = importlib.import_module("scripts.build-all")
+upload = importlib.import_module("scripts.upload")
+
+
+class BuildAllTest(unittest.TestCase):
+    def write_channel(self, channel: Path, packages: list[dict]) -> None:
+        for platform in ("linux-64", "noarch"):
+            directory = channel / platform
+            directory.mkdir(parents=True)
+            records = {}
+            for package in packages if platform == "linux-64" else []:
+                record = {
+                    **package,
+                    "build": "0",
+                    "build_number": 0,
+                    "subdir": platform,
+                }
+                filename = f"{record['name']}-{record['version']}-0.tar.bz2"
+                with tarfile.open(directory / filename, "w:bz2") as archive:
+                    data = json.dumps(record).encode()
+                    info = tarfile.TarInfo("info/index.json")
+                    info.size = len(data)
+                    archive.addfile(info, io.BytesIO(data))
+                records[filename] = record
+            (directory / "repodata.json").write_text(
+                json.dumps({"info": {"subdir": platform}, "packages": records})
+            )
+
+    def test_other_recipe_output_does_not_shadow_compatible_published_build(
+        self,
+    ) -> None:
+        rattler = shutil.which("rattler-build")
+        self.assertIsNotNone(rattler, "Run tests in the Pixi environment")
+        with tempfile.TemporaryDirectory() as temp, chdir(temp):
+            root = Path(temp).resolve()
+            self.write_channel(
+                root / "published",
+                [
+                    {"name": "mojo-compiler", "version": "1.0.0", "depends": []},
+                    {
+                        "name": "mojo-snappy",
+                        "version": "0.2.0",
+                        "depends": ["mojo-compiler ==1.0.0"],
+                    },
+                ],
+            )
+            incompatible = [
+                {
+                    "name": "mojo-snappy",
+                    "version": "0.2.0",
+                    "depends": ["mojo-compiler ==1.1.0"],
+                }
+            ]
+            # Cover both the previous shared layout and another recipe's output.
+            self.write_channel(root / "output", incompatible)
+            self.write_channel(root / "output" / "mojo-snappy", incompatible)
+            recipe = root / "recipes" / "pyroquet" / "recipe.yaml"
+            recipe.parent.mkdir(parents=True)
+            recipe.write_text(
+                'package:\n  name: pyroquet\n  version: "0.1.1"\n'
+                "requirements:\n  host:\n"
+                "    - mojo-compiler ==1.0.0\n    - mojo-snappy ==0.2.0\n"
+            )
+            variants = root / "variants" / "variants.yaml"
+            variants.parent.mkdir()
+            variants.write_text("{}\n")
+            results = []
+
+            def solve(command: list[str]) -> subprocess.CompletedProcess[str]:
+                # Use the real command assembled by build-all, but only solve:
+                # no compiler execution or network packages are needed.
+                result = subprocess.run(
+                    [
+                        rattler,
+                        *command[1:],
+                        "--render-only",
+                        "--with-solve",
+                        "--target-platform=linux-64",
+                        "--build-platform=linux-64",
+                        "--color=never",
+                    ],
+                    capture_output=True,
+                    text=True,
+                )
+                results.append(result)
+                return result
+
+            with (
+                patch.object(
+                    build_all, "DEFAULT_CHANNELS", [(root / "published").as_uri()]
+                ),
+                patch.object(build_all, "recipe_name_collisions", return_value=False),
+                patch.object(build_all, "run_command_unchecked", side_effect=solve),
+                patch("sys.argv", ["build-all"]),
+                patch.dict(os.environ, {"CONDA_BLD_PATH": str(root / "output")}),
+            ):
+                os.environ.pop("DATA_FILE", None)
+                with self.assertRaises(SystemExit) as exited:
+                    build_all.main()
+            self.assertEqual(len(results), 1)
+            self.assertEqual(exited.exception.code, 0, results[0].stderr)
+
+    def test_upload_discovers_packages_in_recipe_output_directories(self) -> None:
+        with tempfile.TemporaryDirectory() as temp, chdir(temp):
+            package = Path("output/mojo-snappy/linux-64/mojo-snappy-0.2.0-0.conda")
+            package.parent.mkdir(parents=True)
+            package.touch()
+            with (
+                patch("sys.argv", ["upload", "--channel", "test-channel"]),
+                patch.object(upload, "run_command_unchecked") as run,
+            ):
+                run.return_value.returncode = 0
+                with self.assertRaises(SystemExit) as exited:
+                    upload.main()
+            self.assertEqual(exited.exception.code, 0)
+            self.assertEqual(run.call_args.args[0][-1], str(package))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_commit_changes.py

@@ -0,0 +1,88 @@
+"""Exercise compatibility commits against a local Git remote."""
+
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class CommitChangesTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.remote = self.root / "remote.git"
+        self.writer = self.root / "writer"
+        self.runner = self.root / "runner"
+        self.git(self.root, "init", "--bare", str(self.remote))
+        self.git(self.root, "clone", str(self.remote), str(self.writer))
+        self.configure(self.writer)
+        self.git(self.writer, "checkout", "-b", "main")
+        (self.writer / "compatibility.json").write_text("{}\n")
+        self.git(self.writer, "add", ".")
+        self.git(self.writer, "commit", "-m", "Initial data")
+        self.git(self.writer, "push", "origin", "main")
+        self.git(
+            self.root, "clone", "--branch", "main", str(self.remote), str(self.runner)
+        )
+        self.configure(self.runner)
+
+    def git(self, cwd: Path, *args: str) -> str:
+        return subprocess.run(
+            ["git", *args], cwd=cwd, check=True, capture_output=True, text=True
+        ).stdout.strip()
+
+    def configure(self, cwd: Path) -> None:
+        self.git(cwd, "config", "user.name", "Test")
+        self.git(cwd, "config", "user.email", "test@example.invalid")
+        self.git(cwd, "config", "pull.rebase", "false")
+        self.git(cwd, "config", "commit.gpgsign", "false")
+
+    def run_script(self) -> subprocess.CompletedProcess[str]:
+        return subprocess.run(
+            [sys.executable, "-m", "scripts.commit-changes", "compatibility.json"],
+            cwd=self.runner,
+            env={**os.environ, "PYTHONPATH": str(ROOT)},
+            capture_output=True,
+            text=True,
+        )
+
+    def test_remote_already_has_generated_update(self) -> None:
+        update = '{"example": {"failed_at": "2026-10-18"}}\n'
+        (self.runner / "compatibility.json").write_text(update)
+        (self.writer / "compatibility.json").write_text(update)
+        self.git(self.writer, "add", ".")
+        self.git(self.writer, "commit", "-m", "Concurrent update")
+        self.git(self.writer, "push", "origin", "main")
+        result = self.run_script()
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        self.assertEqual(self.git(self.runner, "status", "--porcelain"), "")
+        self.assertEqual(
+            self.git(self.runner, "rev-parse", "HEAD"),
+            self.git(self.writer, "rev-parse", "HEAD"),
+        )
+
+    def test_new_update_is_committed_and_pushed(self) -> None:
+        (self.runner / "compatibility.json").write_text('{"new": {}}\n')
+        result = self.run_script()
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        self.assertEqual(self.git(self.runner, "status", "--porcelain"), "")
+        self.assertEqual(
+            self.git(self.runner, "rev-parse", "HEAD"),
+            self.git(self.root, "--git-dir", str(self.remote), "rev-parse", "main"),
+        )
+
+    def test_unchanged_data_is_a_successful_noop(self) -> None:
+        before = self.git(self.runner, "rev-parse", "HEAD")
+        result = self.run_script()
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        self.assertEqual(self.git(self.runner, "rev-parse", "HEAD"), before)
+
+
+if __name__ == "__main__":
+    unittest.main()