From a830d7002d8668aaf6045de96fd1b0186ecbfece Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 09:41:47 +0000 Subject: [PATCH 1/3] feat: normalize line endings to LF in noarch packages Text files that are checked out or generated on Windows often carry CRLF (or bare CR) line endings. When those files end up in a `noarch` package, the resulting archive differs from one built on Unix, breaking reproducibility across platforms (issue #837). `write_to_dest` now normalizes UTF-8 text file line endings to LF for any `noarch` package (both `generic` and `python`, gated on `target_platform == Platform::NoArch`). To stay safe, only valid UTF-8 text is rewritten; binary files and other encodings (e.g. UTF-16/UTF-32, where a naive byte-level CRLF rewrite could corrupt multi-byte code units) are copied verbatim, as are files that already use LF exclusively (which also preserves the fast copy path and file metadata). File permissions are preserved when a file is rewritten. Uses the existing `line-ending` crate for normalization. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015wnV3gL96hhbBuUd7hjejY --- CHANGELOG.md | 1 + .../src/packaging/file_mapper.rs | 105 +++++++++++++++++- 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e612113d8..f3568de4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The `source.filter` field is now also supported for `url` and `git` sources (it was previously only available for `path` sources). The filter is applied to the files copied into the work directory: the contents of the extracted archive for `url` sources (and `path` sources pointing to an archive) and the checked-out tree for `git` sources. This allows large sources to be trimmed down to the parts needed for the build. - An empty `sha256` or `md5` (e.g. `sha256: ""`) is now accepted and treated as an all-zeros placeholder (`0000...0000`). This makes it easier to scaffold a recipe before the real checksum is known: the build downloads the source and reports the actual checksum in the resulting mismatch. (#2524) - The build summary now prints a section for each `requirements.extras` group (optional dependency group) in the run dependencies table, so the resolved contents of each extra are visible. The section is omitted when the recipe defines no extras. +- Line endings in UTF-8 text files are now normalized to LF when building `noarch` packages (both `generic` and `python`). This makes `noarch` packages reproducible regardless of whether they are built on Windows or Unix. Binary files and other encodings (e.g. UTF-16) are left untouched. (#837) ### Changed diff --git a/crates/rattler_build_core/src/packaging/file_mapper.rs b/crates/rattler_build_core/src/packaging/file_mapper.rs index 572a9e255..52e5cffa3 100644 --- a/crates/rattler_build_core/src/packaging/file_mapper.rs +++ b/crates/rattler_build_core/src/packaging/file_mapper.rs @@ -2,6 +2,8 @@ use crate::metadata::Output; use fs_err as fs; +use line_ending::LineEnding; +use rattler_conda_types::Platform; use std::{ collections::HashSet, path::{Component, Path, PathBuf}, @@ -9,6 +11,42 @@ use std::{ use super::PackagingError; +/// Copy `src` to `dst`, normalizing line endings to LF for UTF-8 text files. +/// +/// `noarch` packages must be reproducible regardless of the operating system they are +/// built on. Text files that are checked out or generated on Windows often carry CRLF +/// (or bare CR) line endings, which would otherwise leak into the package and make the +/// resulting archive differ from one built on Unix (see +/// ). +/// +/// To keep the transformation safe we only rewrite files that are valid UTF-8 text. +/// Binary files and other text encodings (e.g. UTF-16/UTF-32, where a naive byte-level +/// CRLF rewrite could corrupt multi-byte code units such as `U+0A0D`) are copied +/// verbatim. Files that already use LF exclusively are also copied verbatim, which +/// preserves the fast copy path and the original file metadata. +fn copy_normalizing_line_endings(src: &Path, dst: &Path) -> Result<(), PackagingError> { + let content = fs::read(src)?; + + // Inspect the leading bytes to decide whether the file looks like text, mirroring + // how content types are detected elsewhere in the packaging code. + let is_text = content_inspector::inspect(&content[..content.len().min(1024)]).is_text(); + + if is_text && let Ok(text) = std::str::from_utf8(&content) { + let normalized = LineEnding::normalize(text); + if normalized.as_bytes() != content.as_slice() { + fs::write(dst, normalized.as_bytes())?; + // Preserve the permissions (e.g. the executable bit) that `fs::copy` + // would otherwise carry over. + fs::set_permissions(dst, fs::metadata(src)?.permissions())?; + return Ok(()); + } + } + + // Binary, non-UTF-8, or already LF-normalized files are copied unchanged. + fs::copy(src, dst)?; + Ok(()) +} + /// We check the (new) `pyc` files against the old files from the environment. /// This is a temporary measure to avoid packaging `pyc` files that are not /// generated by the build process. @@ -92,6 +130,8 @@ impl Output { /// `Scripts` is replaced with `python-scripts` (on Windows only). All other files are included /// as-is. /// * Absolute symlinks are made relative so that they are easily relocatable. + /// * For any `noarch` package (generic or python), UTF-8 text file line endings are + /// normalized to LF so that packages are reproducible across Windows and Unix builds. pub fn write_to_dest( &self, path: &Path, @@ -274,7 +314,13 @@ impl Output { Ok(None) } else { tracing::trace!("Copying file {:?} to {:?}", path, dest_path); - fs::copy(path, &dest_path)?; + if target_platform == &Platform::NoArch { + // Normalize text-file line endings to LF for noarch packages so that + // builds are reproducible across Windows and Unix (issue #837). + copy_normalizing_line_endings(path, &dest_path)?; + } else { + fs::copy(path, &dest_path)?; + } Ok(Some(dest_path)) } } @@ -289,6 +335,63 @@ mod test { use crate::packaging::file_mapper::filter_pyc; + #[test] + fn test_copy_normalizing_line_endings() { + let temp_dir = tempfile::tempdir().unwrap(); + + // (name, input, expected output) — all line endings collapse to LF. + let cases: &[(&str, &[u8], &[u8])] = &[ + ("crlf.txt", b"line1\r\nline2\r\n", b"line1\nline2\n"), + ("cr.txt", b"line1\rline2\r", b"line1\nline2\n"), + ("lf.txt", b"line1\nline2\n", b"line1\nline2\n"), + ("mixed.txt", b"a\r\nb\rc\n", b"a\nb\nc\n"), + ]; + for (name, input, expected) in cases { + let src = temp_dir.path().join(name); + let dst = temp_dir.path().join(format!("out_{name}")); + std::fs::write(&src, input).unwrap(); + super::copy_normalizing_line_endings(&src, &dst).unwrap(); + assert_eq!(&std::fs::read(&dst).unwrap(), expected, "text case {name}"); + } + + // Binary data (contains a NUL and a CR/LF byte pair) must be copied verbatim. + let binary: &[u8] = &[0x00, 0x01, 0x02, 0x0D, 0x0A, 0xFF]; + let src = temp_dir.path().join("data.bin"); + let dst = temp_dir.path().join("data_out.bin"); + std::fs::write(&src, binary).unwrap(); + super::copy_normalizing_line_endings(&src, &dst).unwrap(); + assert_eq!(std::fs::read(&dst).unwrap(), binary, "binary preserved"); + + // UTF-16LE (BOM `FF FE`): `U+0A0D` encodes as bytes `0D 0A`. A naive byte-level + // CRLF rewrite would corrupt it, so non-UTF-8 encodings are copied verbatim. + let utf16: &[u8] = &[0xFF, 0xFE, 0x0D, 0x0A, 0x41, 0x00]; + let src = temp_dir.path().join("u16.txt"); + let dst = temp_dir.path().join("u16_out.txt"); + std::fs::write(&src, utf16).unwrap(); + super::copy_normalizing_line_endings(&src, &dst).unwrap(); + assert_eq!(std::fs::read(&dst).unwrap(), utf16, "utf-16 preserved"); + } + + #[test] + #[cfg(unix)] + fn test_copy_normalizing_line_endings_preserves_mode() { + use std::os::unix::fs::PermissionsExt; + + let temp_dir = tempfile::tempdir().unwrap(); + let src = temp_dir.path().join("script.sh"); + std::fs::write(&src, b"#!/bin/sh\r\necho hi\r\n").unwrap(); + std::fs::set_permissions(&src, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let dst = temp_dir.path().join("script_out.sh"); + super::copy_normalizing_line_endings(&src, &dst).unwrap(); + + // The file went through the rewrite path (CRLF was normalized) ... + assert_eq!(std::fs::read(&dst).unwrap(), b"#!/bin/sh\necho hi\n"); + // ... and the executable bit was preserved. + let mode = std::fs::metadata(&dst).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o755); + } + #[test] fn test_filter_file() { let test_cases = vec![ From 6217f455528e532a6bc9fe15eeb37d074ebee93f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 10:23:18 +0000 Subject: [PATCH 2/3] perf: avoid reading non-text/unchanged files into memory Classify each file from its first kilobyte and only pull UTF-8 text files that actually contain a carriage return fully into memory. Binary files, other encodings, and LF-only text now go straight through `fs::copy` without an intermediate in-memory buffer, and the redundant normalized-vs-original comparison is replaced by a cheap `memchr` scan for '\r'. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015wnV3gL96hhbBuUd7hjejY --- .../src/packaging/file_mapper.rs | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/crates/rattler_build_core/src/packaging/file_mapper.rs b/crates/rattler_build_core/src/packaging/file_mapper.rs index 52e5cffa3..d855ab309 100644 --- a/crates/rattler_build_core/src/packaging/file_mapper.rs +++ b/crates/rattler_build_core/src/packaging/file_mapper.rs @@ -19,22 +19,40 @@ use super::PackagingError; /// resulting archive differ from one built on Unix (see /// ). /// -/// To keep the transformation safe we only rewrite files that are valid UTF-8 text. -/// Binary files and other text encodings (e.g. UTF-16/UTF-32, where a naive byte-level -/// CRLF rewrite could corrupt multi-byte code units such as `U+0A0D`) are copied -/// verbatim. Files that already use LF exclusively are also copied verbatim, which -/// preserves the fast copy path and the original file metadata. +/// To keep the transformation safe (and cheap) we only rewrite UTF-8 text files that +/// actually contain a carriage return. Everything else is streamed straight through +/// `fs::copy` without being read into memory: +/// +/// * Binary files and other text encodings (e.g. UTF-16/UTF-32, where a naive byte-level +/// CRLF rewrite could corrupt multi-byte code units such as `U+0A0D`) are classified +/// from the first kilobyte and copied verbatim. +/// * UTF-8 files that already use LF exclusively are copied verbatim too, which preserves +/// the fast copy path and the original file metadata. fn copy_normalizing_line_endings(src: &Path, dst: &Path) -> Result<(), PackagingError> { - let content = fs::read(src)?; - - // Inspect the leading bytes to decide whether the file looks like text, mirroring - // how content types are detected elsewhere in the packaging code. - let is_text = content_inspector::inspect(&content[..content.len().min(1024)]).is_text(); - - if is_text && let Ok(text) = std::str::from_utf8(&content) { - let normalized = LineEnding::normalize(text); - if normalized.as_bytes() != content.as_slice() { - fs::write(dst, normalized.as_bytes())?; + use std::io::Read; + + // Classify the file from its first kilobyte so that large binary files are never + // read fully into memory just to be copied back out unchanged. + let mut file = fs::File::open(src)?; + let mut buffer = vec![0u8; 1024]; + let read = file.read(&mut buffer)?; + buffer.truncate(read); + let content_type = content_inspector::inspect(&buffer); + + // Only genuine UTF-8 (with or without BOM) is a candidate for normalization. + if matches!( + content_type, + content_inspector::ContentType::UTF_8 | content_inspector::ContentType::UTF_8_BOM + ) { + // Read the rest of the file, reusing the bytes we already inspected. + file.read_to_end(&mut buffer)?; + + // Only touch files that actually contain a carriage return, and only if the + // whole file is valid UTF-8 (the leading kilobyte can be misleading). + if memchr::memchr(b'\r', &buffer).is_some() + && let Ok(text) = std::str::from_utf8(&buffer) + { + fs::write(dst, LineEnding::normalize(text).as_bytes())?; // Preserve the permissions (e.g. the executable bit) that `fs::copy` // would otherwise carry over. fs::set_permissions(dst, fs::metadata(src)?.permissions())?; @@ -42,7 +60,7 @@ fn copy_normalizing_line_endings(src: &Path, dst: &Path) -> Result<(), Packaging } } - // Binary, non-UTF-8, or already LF-normalized files are copied unchanged. + // Binary, non-UTF-8, or already LF-only files are copied unchanged. fs::copy(src, dst)?; Ok(()) } From a9f225e8d6997f33f9afb41b850f583f3df8f20c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 12:08:32 +0000 Subject: [PATCH 3/3] fix: use fs_err instead of std::fs in tests to satisfy clippy The project's clippy config disallows std::fs::{read,write,metadata, set_permissions}. Switch the new test helpers to the fs_err alias used throughout the crate. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_015wnV3gL96hhbBuUd7hjejY --- .../src/packaging/file_mapper.rs | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/crates/rattler_build_core/src/packaging/file_mapper.rs b/crates/rattler_build_core/src/packaging/file_mapper.rs index d855ab309..05f3a2806 100644 --- a/crates/rattler_build_core/src/packaging/file_mapper.rs +++ b/crates/rattler_build_core/src/packaging/file_mapper.rs @@ -351,6 +351,8 @@ mod test { path::{Path, PathBuf}, }; + use fs_err as fs; + use crate::packaging::file_mapper::filter_pyc; #[test] @@ -367,27 +369,27 @@ mod test { for (name, input, expected) in cases { let src = temp_dir.path().join(name); let dst = temp_dir.path().join(format!("out_{name}")); - std::fs::write(&src, input).unwrap(); + fs::write(&src, input).unwrap(); super::copy_normalizing_line_endings(&src, &dst).unwrap(); - assert_eq!(&std::fs::read(&dst).unwrap(), expected, "text case {name}"); + assert_eq!(&fs::read(&dst).unwrap(), expected, "text case {name}"); } // Binary data (contains a NUL and a CR/LF byte pair) must be copied verbatim. let binary: &[u8] = &[0x00, 0x01, 0x02, 0x0D, 0x0A, 0xFF]; let src = temp_dir.path().join("data.bin"); let dst = temp_dir.path().join("data_out.bin"); - std::fs::write(&src, binary).unwrap(); + fs::write(&src, binary).unwrap(); super::copy_normalizing_line_endings(&src, &dst).unwrap(); - assert_eq!(std::fs::read(&dst).unwrap(), binary, "binary preserved"); + assert_eq!(fs::read(&dst).unwrap(), binary, "binary preserved"); // UTF-16LE (BOM `FF FE`): `U+0A0D` encodes as bytes `0D 0A`. A naive byte-level // CRLF rewrite would corrupt it, so non-UTF-8 encodings are copied verbatim. let utf16: &[u8] = &[0xFF, 0xFE, 0x0D, 0x0A, 0x41, 0x00]; let src = temp_dir.path().join("u16.txt"); let dst = temp_dir.path().join("u16_out.txt"); - std::fs::write(&src, utf16).unwrap(); + fs::write(&src, utf16).unwrap(); super::copy_normalizing_line_endings(&src, &dst).unwrap(); - assert_eq!(std::fs::read(&dst).unwrap(), utf16, "utf-16 preserved"); + assert_eq!(fs::read(&dst).unwrap(), utf16, "utf-16 preserved"); } #[test] @@ -397,16 +399,16 @@ mod test { let temp_dir = tempfile::tempdir().unwrap(); let src = temp_dir.path().join("script.sh"); - std::fs::write(&src, b"#!/bin/sh\r\necho hi\r\n").unwrap(); - std::fs::set_permissions(&src, std::fs::Permissions::from_mode(0o755)).unwrap(); + fs::write(&src, b"#!/bin/sh\r\necho hi\r\n").unwrap(); + fs::set_permissions(&src, std::fs::Permissions::from_mode(0o755)).unwrap(); let dst = temp_dir.path().join("script_out.sh"); super::copy_normalizing_line_endings(&src, &dst).unwrap(); // The file went through the rewrite path (CRLF was normalized) ... - assert_eq!(std::fs::read(&dst).unwrap(), b"#!/bin/sh\necho hi\n"); + assert_eq!(fs::read(&dst).unwrap(), b"#!/bin/sh\necho hi\n"); // ... and the executable bit was preserved. - let mode = std::fs::metadata(&dst).unwrap().permissions().mode(); + let mode = fs::metadata(&dst).unwrap().permissions().mode(); assert_eq!(mode & 0o777, 0o755); }