feat: support source.filter for URL and archive path sources

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- The `source.filter` field is now also supported for `url` and `git` sources (it was previously only available for `path` sources). The filter is applied to the files copied into the work directory: the contents of the extracted archive for `url` sources (and `path` sources pointing to an archive) and the checked-out tree for `git` sources. This allows large sources to be trimmed down to the parts needed for the build.
+
 ### Changed
 
 - Improve CRAN (R) recipe generation: pass `${R_ARGS}` to `R CMD INSTALL`, add `cross-r-base` for cross-compilation, set `rpaths` for compiled packages, mark pure-R packages as `noarch: generic`, reference `r-base`'s bundled license files, and fix the `r-base` version constraint so it is applied to both `host` and `run` without duplication.

crates/rattler_build_core/src/source/mod.rs

@@ -11,7 +11,7 @@ use crate::{
 };
 
 use fs_err as fs;
-use rattler_build_recipe::stage1::{Source, source::GitRev};
+use rattler_build_recipe::stage1::{GlobVec, Source, source::GitRev};
 use rattler_build_source_cache::{Checksum, cache::is_tarball};
 use rattler_build_source_cache::{
     GitSource as CacheGitSource, Source as CacheSource, UrlSource as CacheUrlSource,
@@ -90,6 +90,7 @@ fn copy_from_cache(
     cache_path: &Path,
     dest_dir: &Path,
     file_name: Option<&str>,
+    filter: &GlobVec,
     tool_config: &tool_configuration::Configuration,
 ) -> Result<(), SourceError> {
     if cache_path.is_dir() {
@@ -103,6 +104,7 @@ fn copy_from_cache(
             || {
                 copy_dir::CopyDir::new(cache_path, dest_dir)
                     .use_gitignore(false)
+                    .with_globvec(filter)
                     .run()
             },
         )?;
@@ -272,6 +274,7 @@ async fn fetch_source(
                 || {
                     copy_dir::CopyDir::new(&result.path, &dest_dir)
                         .use_gitignore(false)
+                        .with_globvec(&git_src.filter)
                         .run()
                 },
             )?;
@@ -316,7 +319,13 @@ async fn fetch_source(
             fs::create_dir_all(&dest_dir)?;
 
             let extracted_path = if result.path.is_dir() {
-                copy_from_cache(&result.path, &dest_dir, None, tool_configuration)?;
+                copy_from_cache(
+                    &result.path,
+                    &dest_dir,
+                    None,
+                    &url_src.filter,
+                    tool_configuration,
+                )?;
 
                 // Track the extracted path for create-patch functionality
                 result
@@ -335,6 +344,7 @@ async fn fetch_source(
                     &result.path,
                     &dest_dir,
                     Some(&file_name),
+                    &url_src.filter,
                     tool_configuration,
                 )?;
                 None
@@ -409,6 +419,7 @@ async fn fetch_source(
                         file_name: None,
                         target_directory: path_src.target_directory.clone(),
                         attestation: None,
+                        filter: path_src.filter.clone(),
                     };
 
                     let cache_url_source = convert_url_source(&temp_url_source)?;
@@ -421,6 +432,7 @@ async fn fetch_source(
                         &result.path,
                         &dest_dir,
                         Some(&file_name_string),
+                        &temp_url_source.filter,
                         tool_configuration,
                     )?;
                 } else {

crates/rattler_build_recipe/src/stage0/evaluate.rs

@@ -2242,6 +2242,7 @@ impl Evaluate for Stage0GitSource {
                 .as_ref()
                 .map(|v| evaluate_string_value(v, context))
                 .transpose()?,
+            filter: evaluate_glob_vec(&self.filter, context)?,
         })
     }
 }
@@ -2295,6 +2296,7 @@ impl Evaluate for Stage0UrlSource {
                 .map(|v| v.evaluate(context))
                 .transpose()?,
             attestation,
+            filter: evaluate_glob_vec(&self.filter, context)?,
         })
     }
 }

crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_full_snapshot.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/rattler_build_recipe/src/stage0/parser/snapshot_tests.rs
-assertion_line: 112
 expression: "serde_yaml::to_string(&recipe).unwrap()"
 ---
 schema_version: 1
@@ -14,6 +13,7 @@ source:
 - url:
   - https://github.com/example/project/archive/v${{ version }}.tar.gz
   sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  filter: []
 build:
   number: ${{ build_number }}
   script: {}

crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_minimal_snapshot.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/rattler_build_recipe/src/stage0/parser/snapshot_tests.rs
-assertion_line: 104
 expression: "serde_yaml::to_string(&recipe).unwrap()"
 ---
 schema_version: 1
@@ -11,6 +10,7 @@ source:
 - url:
   - https://example.com/source.tar.gz
   sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  filter: []
 build:
   script: {}
   python:
@@ -46,6 +46,7 @@ outputs:
   - url:
     - https://example.com/build-source.tar.gz
     sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+    filter: []
   requirements:
     build:
     - gcc

crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_top_level_inherit_snapshot.snap

@@ -1,6 +1,5 @@
 ---
 source: crates/rattler_build_recipe/src/stage0/parser/snapshot_tests.rs
-assertion_line: 136
 expression: "serde_yaml::to_string(&recipe).unwrap()"
 ---
 schema_version: 1
@@ -10,6 +9,7 @@ source:
 - url:
   - https://example.com/top-level-source.tar.gz
   sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+  filter: []
 build:
   number: 0
   script:

crates/rattler_build_recipe/src/stage0/parser/source.rs

@@ -276,6 +276,7 @@ fn parse_git_source(
     let mut lfs = None;
     let mut submodules = None;
     let mut expected_commit = None;
+    let mut filter = IncludeExclude::default();
 
     for (key_node, value_node) in mapping.iter() {
         let key = key_node.as_str();
@@ -320,14 +321,17 @@ fn parse_git_source(
             "expected_commit" => {
                 expected_commit = Some(parse_value(value_node)?);
             }
+            "filter" => {
+                filter = parse_source_filter(value_node)?;
+            }
             _ => {
                 return Err(ParseError::invalid_value(
                     "git source",
                     format!("unknown field '{}'", key),
                     *key_node.span(),
                 )
                 .with_suggestion(
-                    "Valid fields are: git, rev, tag, branch, depth, patches, target_directory, lfs, submodules, expected_commit",
+                    "Valid fields are: git, rev, tag, branch, depth, patches, target_directory, lfs, submodules, expected_commit, filter",
                 ));
             }
         }
@@ -361,6 +365,7 @@ fn parse_git_source(
         lfs,
         submodules,
         expected_commit,
+        filter,
     })
 }
 
@@ -417,6 +422,7 @@ fn parse_url_source(
     let mut patches = ConditionalList::default();
     let mut target_directory = None;
     let mut attestation = None;
+    let mut filter = IncludeExclude::default();
 
     for (key_node, value_node) in mapping.iter() {
         let key = key_node.as_str();
@@ -458,14 +464,17 @@ fn parse_url_source(
                     ));
                 }
             }
+            "filter" => {
+                filter = parse_source_filter(value_node)?;
+            }
             _ => {
                 return Err(ParseError::invalid_value(
                     "url source",
                     format!("unknown field '{}'", key),
                     *key_node.span(),
                 )
                 .with_suggestion(
-                    "Valid fields are: url, sha256, md5, file_name, patches, target_directory, attestation",
+                    "Valid fields are: url, sha256, md5, file_name, patches, target_directory, attestation, filter",
                 ));
             }
         }
@@ -486,6 +495,7 @@ fn parse_url_source(
         patches,
         target_directory,
         attestation,
+        filter,
     })
 }
 

crates/rattler_build_recipe/src/stage0/source.rs

@@ -78,6 +78,10 @@ pub struct GitSource {
     /// Optionally an expected commit hash to verify after checkout
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub expected_commit: Option<Value<String>>,
+
+    /// Filter for files to include/exclude from the checked-out tree
+    #[serde(default)]
+    pub filter: IncludeExclude,
 }
 
 /// Attestation verification configuration
@@ -156,6 +160,10 @@ pub struct UrlSource {
     /// Optional attestation verification configuration
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub attestation: Option<AttestationConfig>,
+
+    /// Filter for files to include/exclude from the extracted source
+    #[serde(default)]
+    pub filter: IncludeExclude,
 }
 
 /// A local path source
@@ -314,6 +322,7 @@ impl GitSource {
             lfs,
             submodules,
             expected_commit,
+            filter,
         } = self;
 
         let mut vars = Vec::new();
@@ -344,6 +353,7 @@ impl GitSource {
         if let Some(ec) = expected_commit {
             vars.extend(ec.used_variables());
         }
+        vars.extend(filter.used_variables());
         vars.sort();
         vars.dedup();
         vars
@@ -361,6 +371,7 @@ impl UrlSource {
             patches,
             target_directory,
             attestation,
+            filter,
         } = self;
 
         let mut vars = Vec::new();
@@ -386,6 +397,7 @@ impl UrlSource {
         if let Some(attestation) = attestation {
             vars.extend(attestation.used_variables());
         }
+        vars.extend(filter.used_variables());
         vars.sort();
         vars.dedup();
         vars

crates/rattler_build_recipe/src/stage1/source.rs

@@ -145,6 +145,10 @@ pub struct GitSource {
     /// Optionally an expected commit hash to verify after checkout
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub expected_commit: Option<String>,
+
+    /// Only take certain files from the checked-out tree (validated glob patterns)
+    #[serde(default, skip_serializing_if = "GlobVec::is_empty")]
+    pub filter: GlobVec,
 }
 
 fn is_false(value: &bool) -> bool {
@@ -323,6 +327,10 @@ pub struct UrlSource {
     /// Optional attestation verification configuration
     #[serde(default, skip_serializing_if = "attestation_is_none_or_empty")]
     pub attestation: Option<AttestationConfig>,
+
+    /// Only take certain files from the extracted source (validated glob patterns)
+    #[serde(default, skip_serializing_if = "GlobVec::is_empty")]
+    pub filter: GlobVec,
 }
 
 /// A local path source (evaluated)