feat: normalize line endings to LF in noarch packages

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - The `source.filter` field is now also supported for `url` and `git` sources (it was previously only available for `path` sources). The filter is applied to the files copied into the work directory: the contents of the extracted archive for `url` sources (and `path` sources pointing to an archive) and the checked-out tree for `git` sources. This allows large sources to be trimmed down to the parts needed for the build.
 - An empty `sha256` or `md5` (e.g. `sha256: ""`) is now accepted and treated as an all-zeros placeholder (`0000...0000`). This makes it easier to scaffold a recipe before the real checksum is known: the build downloads the source and reports the actual checksum in the resulting mismatch. (#2524)
 - The build summary now prints a section for each `requirements.extras` group (optional dependency group) in the run dependencies table, so the resolved contents of each extra are visible. The section is omitted when the recipe defines no extras.
+- Line endings in UTF-8 text files are now normalized to LF when building `noarch` packages (both `generic` and `python`). This makes `noarch` packages reproducible regardless of whether they are built on Windows or Unix. Binary files and other encodings (e.g. UTF-16) are left untouched. (#837)
 
 ### Changed
 

crates/rattler_build_core/src/packaging/file_mapper.rs

@@ -2,13 +2,69 @@
 
 use crate::metadata::Output;
 use fs_err as fs;
+use line_ending::LineEnding;
+use rattler_conda_types::Platform;
 use std::{
     collections::HashSet,
     path::{Component, Path, PathBuf},
 };
 
 use super::PackagingError;
 
+/// Copy `src` to `dst`, normalizing line endings to LF for UTF-8 text files.
+///
+/// `noarch` packages must be reproducible regardless of the operating system they are
+/// built on. Text files that are checked out or generated on Windows often carry CRLF
+/// (or bare CR) line endings, which would otherwise leak into the package and make the
+/// resulting archive differ from one built on Unix (see
+/// <https://github.com/prefix-dev/rattler-build/issues/837>).
+///
+/// To keep the transformation safe (and cheap) we only rewrite UTF-8 text files that
+/// actually contain a carriage return. Everything else is streamed straight through
+/// `fs::copy` without being read into memory:
+///
+/// * Binary files and other text encodings (e.g. UTF-16/UTF-32, where a naive byte-level
+///   CRLF rewrite could corrupt multi-byte code units such as `U+0A0D`) are classified
+///   from the first kilobyte and copied verbatim.
+/// * UTF-8 files that already use LF exclusively are copied verbatim too, which preserves
+///   the fast copy path and the original file metadata.
+fn copy_normalizing_line_endings(src: &Path, dst: &Path) -> Result<(), PackagingError> {
+    use std::io::Read;
+
+    // Classify the file from its first kilobyte so that large binary files are never
+    // read fully into memory just to be copied back out unchanged.
+    let mut file = fs::File::open(src)?;
+    let mut buffer = vec![0u8; 1024];
+    let read = file.read(&mut buffer)?;
+    buffer.truncate(read);
+    let content_type = content_inspector::inspect(&buffer);
+
+    // Only genuine UTF-8 (with or without BOM) is a candidate for normalization.
+    if matches!(
+        content_type,
+        content_inspector::ContentType::UTF_8 | content_inspector::ContentType::UTF_8_BOM
+    ) {
+        // Read the rest of the file, reusing the bytes we already inspected.
+        file.read_to_end(&mut buffer)?;
+
+        // Only touch files that actually contain a carriage return, and only if the
+        // whole file is valid UTF-8 (the leading kilobyte can be misleading).
+        if memchr::memchr(b'\r', &buffer).is_some()
+            && let Ok(text) = std::str::from_utf8(&buffer)
+        {
+            fs::write(dst, LineEnding::normalize(text).as_bytes())?;
+            // Preserve the permissions (e.g. the executable bit) that `fs::copy`
+            // would otherwise carry over.
+            fs::set_permissions(dst, fs::metadata(src)?.permissions())?;
+            return Ok(());
+        }
+    }
+
+    // Binary, non-UTF-8, or already LF-only files are copied unchanged.
+    fs::copy(src, dst)?;
+    Ok(())
+}
+
 /// We check the (new) `pyc` files against the old files from the environment.
 /// This is a temporary measure to avoid packaging `pyc` files that are not
 /// generated by the build process.
@@ -92,6 +148,8 @@ impl Output {
     ///   `Scripts` is replaced with `python-scripts` (on Windows only). All other files are included
     ///   as-is.
     /// * Absolute symlinks are made relative so that they are easily relocatable.
+    /// * For any `noarch` package (generic or python), UTF-8 text file line endings are
+    ///   normalized to LF so that packages are reproducible across Windows and Unix builds.
     pub fn write_to_dest(
         &self,
         path: &Path,
@@ -274,7 +332,13 @@ impl Output {
             Ok(None)
         } else {
             tracing::trace!("Copying file {:?} to {:?}", path, dest_path);
-            fs::copy(path, &dest_path)?;
+            if target_platform == &Platform::NoArch {
+                // Normalize text-file line endings to LF for noarch packages so that
+                // builds are reproducible across Windows and Unix (issue #837).
+                copy_normalizing_line_endings(path, &dest_path)?;
+            } else {
+                fs::copy(path, &dest_path)?;
+            }
             Ok(Some(dest_path))
         }
     }
@@ -287,8 +351,67 @@ mod test {
         path::{Path, PathBuf},
     };
 
+    use fs_err as fs;
+
     use crate::packaging::file_mapper::filter_pyc;
 
+    #[test]
+    fn test_copy_normalizing_line_endings() {
+        let temp_dir = tempfile::tempdir().unwrap();
+
+        // (name, input, expected output) — all line endings collapse to LF.
+        let cases: &[(&str, &[u8], &[u8])] = &[
+            ("crlf.txt", b"line1\r\nline2\r\n", b"line1\nline2\n"),
+            ("cr.txt", b"line1\rline2\r", b"line1\nline2\n"),
+            ("lf.txt", b"line1\nline2\n", b"line1\nline2\n"),
+            ("mixed.txt", b"a\r\nb\rc\n", b"a\nb\nc\n"),
+        ];
+        for (name, input, expected) in cases {
+            let src = temp_dir.path().join(name);
+            let dst = temp_dir.path().join(format!("out_{name}"));
+            fs::write(&src, input).unwrap();
+            super::copy_normalizing_line_endings(&src, &dst).unwrap();
+            assert_eq!(&fs::read(&dst).unwrap(), expected, "text case {name}");
+        }
+
+        // Binary data (contains a NUL and a CR/LF byte pair) must be copied verbatim.
+        let binary: &[u8] = &[0x00, 0x01, 0x02, 0x0D, 0x0A, 0xFF];
+        let src = temp_dir.path().join("data.bin");
+        let dst = temp_dir.path().join("data_out.bin");
+        fs::write(&src, binary).unwrap();
+        super::copy_normalizing_line_endings(&src, &dst).unwrap();
+        assert_eq!(fs::read(&dst).unwrap(), binary, "binary preserved");
+
+        // UTF-16LE (BOM `FF FE`): `U+0A0D` encodes as bytes `0D 0A`. A naive byte-level
+        // CRLF rewrite would corrupt it, so non-UTF-8 encodings are copied verbatim.
+        let utf16: &[u8] = &[0xFF, 0xFE, 0x0D, 0x0A, 0x41, 0x00];
+        let src = temp_dir.path().join("u16.txt");
+        let dst = temp_dir.path().join("u16_out.txt");
+        fs::write(&src, utf16).unwrap();
+        super::copy_normalizing_line_endings(&src, &dst).unwrap();
+        assert_eq!(fs::read(&dst).unwrap(), utf16, "utf-16 preserved");
+    }
+
+    #[test]
+    #[cfg(unix)]
+    fn test_copy_normalizing_line_endings_preserves_mode() {
+        use std::os::unix::fs::PermissionsExt;
+
+        let temp_dir = tempfile::tempdir().unwrap();
+        let src = temp_dir.path().join("script.sh");
+        fs::write(&src, b"#!/bin/sh\r\necho hi\r\n").unwrap();
+        fs::set_permissions(&src, std::fs::Permissions::from_mode(0o755)).unwrap();
+
+        let dst = temp_dir.path().join("script_out.sh");
+        super::copy_normalizing_line_endings(&src, &dst).unwrap();
+
+        // The file went through the rewrite path (CRLF was normalized) ...
+        assert_eq!(fs::read(&dst).unwrap(), b"#!/bin/sh\necho hi\n");
+        // ... and the executable bit was preserved.
+        let mode = fs::metadata(&dst).unwrap().permissions().mode();
+        assert_eq!(mode & 0o777, 0o755);
+    }
+
     #[test]
     fn test_filter_file() {
         let test_cases = vec![