fix(utils): length-prefix paths in compute_content_hash; deprecate bare content_sha* recipe keys

conda_build/metadata.py

@@ -617,9 +617,14 @@ def parse(data, config, path=None):
         "sha256": None,
         "sha384": None,
         "sha512": None,
+        # Deprecated: use the original CEP-19 algorithm (no length-prefixing). Migrate to
+        # content_sha*_v2 keys which use the fixed algorithm.
         "content_sha256": None,
         "content_sha384": None,
         "content_sha512": None,
+        "content_sha256_v2": None,
+        "content_sha384_v2": None,
+        "content_sha512_v2": None,
         "content_hash_skip": list,
         "path": str,
         "path_via_symlink": None,

conda_build/source.py

@@ -49,6 +49,7 @@
 ext_re = re.compile(r"(.*?)(\.(?:tar\.)?[^.]+)$")
 ACCEPTED_HASH_TYPES = ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")
 CONTENT_HASH_KEYS = ("content_sha256", "content_sha384", "content_sha512")
+CONTENT_HASH_KEYS_V2 = ("content_sha256_v2", "content_sha384_v2", "content_sha512_v2")
 
 
 def append_hash_to_fn(fn, hash_value):
@@ -1116,25 +1117,36 @@ def provide(metadata):
                 if not isdir(src_dir):
                     os.makedirs(src_dir)
 
-            for hash_type in CONTENT_HASH_KEYS:
-                if hash_type in source_dict:
-                    expected_content_hash = source_dict[hash_type]
-                    if expected_content_hash in (None, ""):
-                        raise ValueError(
-                            f"Empty {hash_type} hash provided for source item #{idx}"
-                        )
-                    algorithm = hash_type[len("content_") :]
-                    obtained_content_hash = compute_content_hash(
-                        src_dir,
-                        algorithm,
-                        skip=ensure_list(source_dict.get("content_hash_skip") or ()),
-                    )
-                    if expected_content_hash != obtained_content_hash:
-                        raise RuntimeError(
-                            f"{hash_type} mismatch in source item #{idx}: "
-                            f"obtained '{obtained_content_hash}' != "
-                            f"expected '{expected_content_hash}'"
+            skip = ensure_list(source_dict.get("content_hash_skip") or ())
+
+            def _check_content_hashes(content_hash_keys, legacy=False):
+                for hash_type in content_hash_keys:
+                    if hash_type in source_dict:
+                        expected_content_hash = source_dict[hash_type]
+                        if expected_content_hash in (None, ""):
+                            raise ValueError(
+                                f"Empty {hash_type} hash provided for source item #{idx}"
+                            )
+                        if legacy:
+                            algorithm = hash_type[len("content_") :]
+                        else:
+                            algorithm = hash_type[len("content_") : -len("_v2")]
+                        obtained_content_hash = compute_content_hash(
+                            src_dir,
+                            algorithm,
+                            skip=skip,
+                            legacy=legacy,
                         )
+                        if expected_content_hash != obtained_content_hash:
+                            raise RuntimeError(
+                                f"{hash_type} mismatch in source item #{idx}: "
+                                f"obtained '{obtained_content_hash}' != "
+                                f"expected '{expected_content_hash}'"
+                            )
+
+            # Un-versioned keys use the original CEP-19 algorithm (deprecated).
+            _check_content_hashes(CONTENT_HASH_KEYS, legacy=True)
+            _check_content_hashes(CONTENT_HASH_KEYS_V2)
             patches = ensure_list(source_dict.get("patches", []))
             patch_attributes_output = []
             for patch in patches:

conda_build/utils.py

@@ -21,6 +21,7 @@
 import time
 import urllib.parse as urlparse
 import urllib.request as urllib
+import warnings
 from collections import OrderedDict, defaultdict
 from collections.abc import Iterable
 from functools import cache, partial
@@ -2026,13 +2027,22 @@ def sha256_checksum(filename, buffersize=65536):
 
 
 def compute_content_hash(
-    directory: str | Path, algorithm="sha256", skip: Iterable[str] = ()
+    directory: str | Path,
+    algorithm="sha256",
+    skip: Iterable[str] = (),
+    legacy: bool = False,
 ) -> str:
     """
     Given a directory, recursively scan all its contents (without following symlinks) and sort them
     by their full path. For each entry in the contents table, compute the hash for the concatenated
     bytes of:
 
+    - The number of bytes required to encode the path in UTF-8, written as a decimal ASCII
+      string, followed by a UTF-8 encoded `:` separator. The colon acts as a delimiter between
+      the decimal length number and the path bytes that follow, preventing ambiguity when the
+      path starts with a digit (e.g. length ``12`` concatenated directly with path ``3abc``
+      would be indistinguishable from length ``123`` followed by ``abc``).
+      **Not present when** ``legacy=True`` **(CEP-19 behaviour).**
     - UTF-8 encoded path, relative to the input directory. Backslashes are normalized
       to forward slashes before encoding.
     - Then, depending on the type:
@@ -2041,25 +2051,46 @@ def compute_content_hash(
           - The raw bytes of the file contents, if binary.
           - If it can't be read, error out.
         - For a directory, the UTF-8 bytes of a `D` separator, and nothing else.
-        - For a symlink, the UTF-8 bytes of an `L` separator, followed by the UTF-8 encoded bytes
-          for the path it points to. Backslashes MUST be normalized to forward slashes before
-          encoding.
+        - For a symlink, the UTF-8 bytes of an `L` separator, followed by:
+          - The number of bytes required to encode the target path in UTF-8, written as a
+            decimal ASCII string, followed by a UTF-8 encoded `:` separator (same
+            length-prefix-with-delimiter scheme as above).
+            **Not present when** ``legacy=True`` **(CEP-19 behaviour).**
+          - The UTF-8 encoded bytes for the path it points to. Backslashes MUST be normalized
+            to forward slashes before encoding.
         - For any other types, error out.
     - UTF-8 encoded bytes of the string `-`, as separator.
 
+    The length prefixes prevent path/type/content boundary ambiguity: without them a filename
+    like ``testFhello-world`` would hash identically to a file ``test`` with content ``hello``
+    followed by a file ``world``.
+
     Parameters
     ----------
     directory: The path whose contents will be hashed
     algorithm: Name of the algorithm to be used, as expected by `hashlib.new()`
     skip: iterable of paths that should not be checked. If a path ends with a slash, it's
           interpreted as a directory that won't be traversed. It matches the relative paths
           already slashed-normalized (i.e. backwards slashes replaced with forward slashes).
+    legacy: When True, use the original CEP-19 algorithm that does **not** length-prefix paths or
+            symlink targets. This is provided for backwards compatibility with hashes stored under
+            the un-versioned ``content_sha*`` recipe keys, which are deprecated. Prefer the
+            ``content_sha*_v2`` recipe keys (legacy=False) for new recipes.
+            Defaults to False (v2 / this CEP algorithm).
 
     Returns
     -------
     str
         The hexdigest of the computed hash, as described above.
     """
+    if legacy:
+        warnings.warn(
+            "The un-versioned content_sha* recipe keys use the original CEP-19 hashing "
+            "algorithm which is susceptible to hash collisions. Migrate to content_sha*_v2 "
+            "keys to use the fixed algorithm.",
+            PendingDeprecationWarning,
+            stacklevel=2,
+        )
     hasher = hashlib.new(algorithm)
     for path in sorted(Path(directory).rglob("*"), key=str):
         relpath = path.relative_to(directory)
@@ -2077,10 +2108,20 @@ def compute_content_hash(
         ):
             continue
         # encode the relative path to directory, for files, dirs and others
-        hasher.update(relpathstr.encode("utf-8"))
+        # Length-prefix the path to avoid ambiguity between path/type/content boundaries.
+        # Without the length prefix, a filename like "testFhello-world" is indistinguishable
+        # from a file "test" with content "hello" followed by a file "world".
+        # legacy=True skips the prefix to reproduce the original CEP-19 algorithm.
+        path_bytes = relpathstr.encode("utf-8")
+        if not legacy:
+            hasher.update(f"{len(path_bytes)}:".encode())
+        hasher.update(path_bytes)
         if path.is_symlink():
             hasher.update(b"L")
-            hasher.update(str(path.readlink()).replace("\\", "/").encode("utf-8"))
+            target_bytes = str(path.readlink()).replace("\\", "/").encode("utf-8")
+            if not legacy:
+                hasher.update(f"{len(target_bytes)}:".encode())
+            hasher.update(target_bytes)
         elif path.is_dir():
             hasher.update(b"D")
         elif path.is_file():

pyproject.toml

@@ -115,11 +115,9 @@ doctest_optionflags = [
   "ELLIPSIS",
 ]
 filterwarnings = [
-  # elevate conda's deprecated warning to an error
-  "error::PendingDeprecationWarning:conda",
+  # elevate conda's deprecated warnings to errors; pending deprecations only warn
   "error::DeprecationWarning:conda",
-  # elevate conda-build's deprecated warning to an error
-  "error::PendingDeprecationWarning:conda_build",
+  # elevate conda-build's deprecated warnings to errors; pending deprecations only warn
   "error::DeprecationWarning:conda_build",
   # ignore numpy.distutils error
   'ignore:\s+`numpy.distutils` is deprecated:DeprecationWarning:conda_build._load_setup_py_data',

tests/test-recipes/metadata/source_url/meta.yaml

@@ -13,9 +13,13 @@ source:
     sha256: a1932d36ac8ea0dcc3a0b7848a090aedc9247d4bcd75fa75e1771c2b2b01f9ff
     sha384: d366de5e995a4ff6ad9266774e483efb91d9c291c0487c5cf0af055a7b48fd58af205c9455a5b2f654d92d7f3f39ef68
     sha512: 33d2c8f8189f0fe8528bef0c32e62a3acd4362285e447680e7f0af16137df9ab45bf12b6928bdaaf99b5a53e71db4d385a0c1d91bdc0b2ad1d0b1a7bc6d790f1
+    # Deprecated: uses the original CEP-19 algorithm. Migrate to content_sha*_v2.
     content_sha256: a884ace5aa3a7e7f5a8b5adeb5cbfa7209f2ae88134d362c8bbca9c82ad2bb06
     content_sha384: 3644cb7e55fb8f6d7328b19da3ec46be6af1e67291cc48948687cf9493d9b2caea3b5a637d1dfc1a19dd2893ddc38d27
     content_sha512: 79a0c5edc29f979b599f0b694c3f0f07cc91e590c2c3fcb9c3f965767bf5a22fe634f0f142c626ef0859249d0242f3d8cc93922cf14e7ba527eedc3e8c8b354e
+    content_sha256_v2: fa18683d70b5b776b017ac0c55b1086a70c7a12584fc1c2fd5166b79f568c687
+    content_sha384_v2: 6e9566990b181ef8a35e51bcbbf4972fd349018bc3279433eb6cc57f8ed1860fc82fd6661396603d1606acc51c21e4aa
+    content_sha512_v2: 95262d5b9f7f654def1ec0b849e76b2354669d748f3cc43b844fc798cbe7fc68bace45c63e8748344c094aee3538c10f628f37f2ebe0c7694ef3a4da54d3080b
     content_hash_skip:
       - constructor/_version.py
   # This is the same tarball but compressed differently. They should have the same content hashes!
@@ -28,18 +32,18 @@ source:
     sha256: 77406614899f5c2e21e2133a774b8470ba75a86e76dda799c2b39bcbce860955
     sha384: e93d217376c86ab374be93c44fa03b05673e23de78033812a8f0620ce1ca6a4082fedd8b2599341ffd8dcfd201479ff4
     sha512: 23e2ef512e43cb3b75637650901d5c86e0edc812a95fe85b19b45feddabe74bd72d6affac30b133c37a69046b3e27635a84107df5f64e403e1b21dc8f56ceedb
-    content_sha256: a884ace5aa3a7e7f5a8b5adeb5cbfa7209f2ae88134d362c8bbca9c82ad2bb06
-    content_sha384: 3644cb7e55fb8f6d7328b19da3ec46be6af1e67291cc48948687cf9493d9b2caea3b5a637d1dfc1a19dd2893ddc38d27
-    content_sha512: 79a0c5edc29f979b599f0b694c3f0f07cc91e590c2c3fcb9c3f965767bf5a22fe634f0f142c626ef0859249d0242f3d8cc93922cf14e7ba527eedc3e8c8b354e
+    content_sha256_v2: fa18683d70b5b776b017ac0c55b1086a70c7a12584fc1c2fd5166b79f568c687
+    content_sha384_v2: 6e9566990b181ef8a35e51bcbbf4972fd349018bc3279433eb6cc57f8ed1860fc82fd6661396603d1606acc51c21e4aa
+    content_sha512_v2: 95262d5b9f7f654def1ec0b849e76b2354669d748f3cc43b844fc798cbe7fc68bace45c63e8748344c094aee3538c10f628f37f2ebe0c7694ef3a4da54d3080b
     content_hash_skip:
       - constructor/_version.py
   # This is the same tag as above, but cloned directly. They should have the same content hashes!
   - folder: constructor-git
     git_url: https://github.com/conda/constructor.git
     git_rev: "3.0.0"
-    content_sha256: a884ace5aa3a7e7f5a8b5adeb5cbfa7209f2ae88134d362c8bbca9c82ad2bb06
-    content_sha384: 3644cb7e55fb8f6d7328b19da3ec46be6af1e67291cc48948687cf9493d9b2caea3b5a637d1dfc1a19dd2893ddc38d27
-    content_sha512: 79a0c5edc29f979b599f0b694c3f0f07cc91e590c2c3fcb9c3f965767bf5a22fe634f0f142c626ef0859249d0242f3d8cc93922cf14e7ba527eedc3e8c8b354e
+    content_sha256_v2: fa18683d70b5b776b017ac0c55b1086a70c7a12584fc1c2fd5166b79f568c687
+    content_sha384_v2: 6e9566990b181ef8a35e51bcbbf4972fd349018bc3279433eb6cc57f8ed1860fc82fd6661396603d1606acc51c21e4aa
+    content_sha512_v2: 95262d5b9f7f654def1ec0b849e76b2354669d748f3cc43b844fc798cbe7fc68bace45c63e8748344c094aee3538c10f628f37f2ebe0c7694ef3a4da54d3080b
     content_hash_skip:
       - .git/
       - constructor/_version.py