diff --git a/CHANGELOG.md b/CHANGELOG.md index 1208288c0..ca54b208c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- The `source.filter` field is now also supported for `url` and `git` sources (it was previously only available for `path` sources). The filter is applied to the files copied into the work directory: the contents of the extracted archive for `url` sources (and `path` sources pointing to an archive) and the checked-out tree for `git` sources. This allows large sources to be trimmed down to the parts needed for the build. + ### Changed - Improve CRAN (R) recipe generation: pass `${R_ARGS}` to `R CMD INSTALL`, add `cross-r-base` for cross-compilation, set `rpaths` for compiled packages, mark pure-R packages as `noarch: generic`, reference `r-base`'s bundled license files, and fix the `r-base` version constraint so it is applied to both `host` and `run` without duplication. diff --git a/crates/rattler_build_core/src/source/mod.rs b/crates/rattler_build_core/src/source/mod.rs index cc817bf0b..9c8be2fbc 100644 --- a/crates/rattler_build_core/src/source/mod.rs +++ b/crates/rattler_build_core/src/source/mod.rs @@ -11,7 +11,7 @@ use crate::{ }; use fs_err as fs; -use rattler_build_recipe::stage1::{Source, source::GitRev}; +use rattler_build_recipe::stage1::{GlobVec, Source, source::GitRev}; use rattler_build_source_cache::{Checksum, cache::is_tarball}; use rattler_build_source_cache::{ GitSource as CacheGitSource, Source as CacheSource, UrlSource as CacheUrlSource, @@ -90,6 +90,7 @@ fn copy_from_cache( cache_path: &Path, dest_dir: &Path, file_name: Option<&str>, + filter: &GlobVec, tool_config: &tool_configuration::Configuration, ) -> Result<(), SourceError> { if cache_path.is_dir() { @@ -103,6 +104,7 @@ fn copy_from_cache( || { copy_dir::CopyDir::new(cache_path, dest_dir) .use_gitignore(false) + .with_globvec(filter) .run() }, )?; @@ -272,6 +274,7 @@ async fn fetch_source( || { copy_dir::CopyDir::new(&result.path, &dest_dir) .use_gitignore(false) + .with_globvec(&git_src.filter) .run() }, )?; @@ -316,7 +319,13 @@ async fn fetch_source( fs::create_dir_all(&dest_dir)?; let extracted_path = if result.path.is_dir() { - copy_from_cache(&result.path, &dest_dir, None, tool_configuration)?; + copy_from_cache( + &result.path, + &dest_dir, + None, + &url_src.filter, + tool_configuration, + )?; // Track the extracted path for create-patch functionality result @@ -335,6 +344,7 @@ async fn fetch_source( &result.path, &dest_dir, Some(&file_name), + &url_src.filter, tool_configuration, )?; None @@ -409,6 +419,7 @@ async fn fetch_source( file_name: None, target_directory: path_src.target_directory.clone(), attestation: None, + filter: path_src.filter.clone(), }; let cache_url_source = convert_url_source(&temp_url_source)?; @@ -421,6 +432,7 @@ async fn fetch_source( &result.path, &dest_dir, Some(&file_name_string), + &temp_url_source.filter, tool_configuration, )?; } else { diff --git a/crates/rattler_build_recipe/src/stage0/evaluate.rs b/crates/rattler_build_recipe/src/stage0/evaluate.rs index f5f8f00b0..65f796b22 100644 --- a/crates/rattler_build_recipe/src/stage0/evaluate.rs +++ b/crates/rattler_build_recipe/src/stage0/evaluate.rs @@ -2242,6 +2242,7 @@ impl Evaluate for Stage0GitSource { .as_ref() .map(|v| evaluate_string_value(v, context)) .transpose()?, + filter: evaluate_glob_vec(&self.filter, context)?, }) } } @@ -2295,6 +2296,7 @@ impl Evaluate for Stage0UrlSource { .map(|v| v.evaluate(context)) .transpose()?, attestation, + filter: evaluate_glob_vec(&self.filter, context)?, }) } } diff --git a/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_full_snapshot.snap b/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_full_snapshot.snap index 1713892b5..f9642879f 100644 --- a/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_full_snapshot.snap +++ b/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_full_snapshot.snap @@ -1,6 +1,5 @@ --- source: crates/rattler_build_recipe/src/stage0/parser/snapshot_tests.rs -assertion_line: 112 expression: "serde_yaml::to_string(&recipe).unwrap()" --- schema_version: 1 @@ -14,6 +13,7 @@ source: - url: - https://github.com/example/project/archive/v${{ version }}.tar.gz sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + filter: [] build: number: ${{ build_number }} script: {} diff --git a/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_minimal_snapshot.snap b/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_minimal_snapshot.snap index 32076d276..e0f158d69 100644 --- a/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_minimal_snapshot.snap +++ b/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_minimal_snapshot.snap @@ -1,6 +1,5 @@ --- source: crates/rattler_build_recipe/src/stage0/parser/snapshot_tests.rs -assertion_line: 104 expression: "serde_yaml::to_string(&recipe).unwrap()" --- schema_version: 1 @@ -11,6 +10,7 @@ source: - url: - https://example.com/source.tar.gz sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + filter: [] build: script: {} python: @@ -46,6 +46,7 @@ outputs: - url: - https://example.com/build-source.tar.gz sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + filter: [] requirements: build: - gcc diff --git a/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_top_level_inherit_snapshot.snap b/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_top_level_inherit_snapshot.snap index 3e025b610..612b73c77 100644 --- a/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_top_level_inherit_snapshot.snap +++ b/crates/rattler_build_recipe/src/stage0/parser/snapshots/rattler_build_recipe__stage0__parser__snapshot_tests__multi_output_top_level_inherit_snapshot.snap @@ -1,6 +1,5 @@ --- source: crates/rattler_build_recipe/src/stage0/parser/snapshot_tests.rs -assertion_line: 136 expression: "serde_yaml::to_string(&recipe).unwrap()" --- schema_version: 1 @@ -10,6 +9,7 @@ source: - url: - https://example.com/top-level-source.tar.gz sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + filter: [] build: number: 0 script: diff --git a/crates/rattler_build_recipe/src/stage0/parser/source.rs b/crates/rattler_build_recipe/src/stage0/parser/source.rs index 5934e0006..6462ee34d 100644 --- a/crates/rattler_build_recipe/src/stage0/parser/source.rs +++ b/crates/rattler_build_recipe/src/stage0/parser/source.rs @@ -276,6 +276,7 @@ fn parse_git_source( let mut lfs = None; let mut submodules = None; let mut expected_commit = None; + let mut filter = IncludeExclude::default(); for (key_node, value_node) in mapping.iter() { let key = key_node.as_str(); @@ -320,6 +321,9 @@ fn parse_git_source( "expected_commit" => { expected_commit = Some(parse_value(value_node)?); } + "filter" => { + filter = parse_source_filter(value_node)?; + } _ => { return Err(ParseError::invalid_value( "git source", @@ -327,7 +331,7 @@ fn parse_git_source( *key_node.span(), ) .with_suggestion( - "Valid fields are: git, rev, tag, branch, depth, patches, target_directory, lfs, submodules, expected_commit", + "Valid fields are: git, rev, tag, branch, depth, patches, target_directory, lfs, submodules, expected_commit, filter", )); } } @@ -361,6 +365,7 @@ fn parse_git_source( lfs, submodules, expected_commit, + filter, }) } @@ -417,6 +422,7 @@ fn parse_url_source( let mut patches = ConditionalList::default(); let mut target_directory = None; let mut attestation = None; + let mut filter = IncludeExclude::default(); for (key_node, value_node) in mapping.iter() { let key = key_node.as_str(); @@ -458,6 +464,9 @@ fn parse_url_source( )); } } + "filter" => { + filter = parse_source_filter(value_node)?; + } _ => { return Err(ParseError::invalid_value( "url source", @@ -465,7 +474,7 @@ fn parse_url_source( *key_node.span(), ) .with_suggestion( - "Valid fields are: url, sha256, md5, file_name, patches, target_directory, attestation", + "Valid fields are: url, sha256, md5, file_name, patches, target_directory, attestation, filter", )); } } @@ -486,6 +495,7 @@ fn parse_url_source( patches, target_directory, attestation, + filter, }) } diff --git a/crates/rattler_build_recipe/src/stage0/source.rs b/crates/rattler_build_recipe/src/stage0/source.rs index 9d0abc64e..0f29841d5 100644 --- a/crates/rattler_build_recipe/src/stage0/source.rs +++ b/crates/rattler_build_recipe/src/stage0/source.rs @@ -78,6 +78,10 @@ pub struct GitSource { /// Optionally an expected commit hash to verify after checkout #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_commit: Option>, + + /// Filter for files to include/exclude from the checked-out tree + #[serde(default)] + pub filter: IncludeExclude, } /// Attestation verification configuration @@ -156,6 +160,10 @@ pub struct UrlSource { /// Optional attestation verification configuration #[serde(default, skip_serializing_if = "Option::is_none")] pub attestation: Option, + + /// Filter for files to include/exclude from the extracted source + #[serde(default)] + pub filter: IncludeExclude, } /// A local path source @@ -314,6 +322,7 @@ impl GitSource { lfs, submodules, expected_commit, + filter, } = self; let mut vars = Vec::new(); @@ -344,6 +353,7 @@ impl GitSource { if let Some(ec) = expected_commit { vars.extend(ec.used_variables()); } + vars.extend(filter.used_variables()); vars.sort(); vars.dedup(); vars @@ -361,6 +371,7 @@ impl UrlSource { patches, target_directory, attestation, + filter, } = self; let mut vars = Vec::new(); @@ -386,6 +397,7 @@ impl UrlSource { if let Some(attestation) = attestation { vars.extend(attestation.used_variables()); } + vars.extend(filter.used_variables()); vars.sort(); vars.dedup(); vars diff --git a/crates/rattler_build_recipe/src/stage1/source.rs b/crates/rattler_build_recipe/src/stage1/source.rs index 00a922d38..ef563307f 100644 --- a/crates/rattler_build_recipe/src/stage1/source.rs +++ b/crates/rattler_build_recipe/src/stage1/source.rs @@ -145,6 +145,10 @@ pub struct GitSource { /// Optionally an expected commit hash to verify after checkout #[serde(default, skip_serializing_if = "Option::is_none")] pub expected_commit: Option, + + /// Only take certain files from the checked-out tree (validated glob patterns) + #[serde(default, skip_serializing_if = "GlobVec::is_empty")] + pub filter: GlobVec, } fn is_false(value: &bool) -> bool { @@ -323,6 +327,10 @@ pub struct UrlSource { /// Optional attestation verification configuration #[serde(default, skip_serializing_if = "attestation_is_none_or_empty")] pub attestation: Option, + + /// Only take certain files from the extracted source (validated glob patterns) + #[serde(default, skip_serializing_if = "GlobVec::is_empty")] + pub filter: GlobVec, } /// A local path source (evaluated) diff --git a/crates/rattler_build_recipe/tests/source_parsing_test.rs b/crates/rattler_build_recipe/tests/source_parsing_test.rs index 4d9299667..c69b7bb74 100644 --- a/crates/rattler_build_recipe/tests/source_parsing_test.rs +++ b/crates/rattler_build_recipe/tests/source_parsing_test.rs @@ -94,6 +94,109 @@ build: } } +#[test] +fn test_parse_recipe_with_git_source_filter() { + let yaml = r#" +package: + name: test + version: 1.0.0 + +source: + git: https://github.com/example/repo.git + tag: v1.0.0 + filter: + include: + - "src/*" + exclude: + - "src/vendor/*" + +build: + number: 0 +"#; + + let recipe = parse_recipe_from_source(yaml).unwrap(); + assert_eq!(recipe.source.len(), 1); + + let source = get_concrete_source(&recipe.source.as_slice()[0]).unwrap(); + match source { + Source::Git(git) => match &git.filter { + IncludeExclude::Mapping { include, exclude } => { + assert_eq!(include.len(), 1); + assert_eq!(exclude.len(), 1); + } + _ => panic!("Expected Mapping variant for filter"), + }, + _ => panic!("Expected Git source"), + } +} + +#[test] +fn test_parse_recipe_with_url_source_filter() { + let yaml = r#" +package: + name: test + version: 1.0.0 + +source: + url: https://example.com/archive.tar.gz + sha256: e03c8123866dd68f129e8a29082011db418ce90863948f563c01b814670782c6 + filter: + - "cmake/*" + - "clang/*" + +build: + number: 0 +"#; + + let recipe = parse_recipe_from_source(yaml).unwrap(); + assert_eq!(recipe.source.len(), 1); + + let source = get_concrete_source(&recipe.source.as_slice()[0]).unwrap(); + match source { + Source::Url(url_src) => match &url_src.filter { + IncludeExclude::List(list) => { + assert_eq!(list.len(), 2); + } + _ => panic!("Expected List variant for filter"), + }, + _ => panic!("Expected URL source"), + } +} + +#[test] +fn test_parse_recipe_with_url_source_filter_include_exclude() { + let yaml = r#" +package: + name: test + version: 1.0.0 + +source: + url: https://example.com/archive.tar.gz + sha256: e03c8123866dd68f129e8a29082011db418ce90863948f563c01b814670782c6 + filter: + exclude: + - "third_party/fmt/*" + +build: + number: 0 +"#; + + let recipe = parse_recipe_from_source(yaml).unwrap(); + assert_eq!(recipe.source.len(), 1); + + let source = get_concrete_source(&recipe.source.as_slice()[0]).unwrap(); + match source { + Source::Url(url_src) => match &url_src.filter { + IncludeExclude::Mapping { include, exclude } => { + assert_eq!(include.len(), 0); + assert_eq!(exclude.len(), 1); + } + _ => panic!("Expected Mapping variant for filter"), + }, + _ => panic!("Expected URL source"), + } +} + #[test] fn test_parse_recipe_with_path_source() { let yaml = r#" diff --git a/docs/reference/recipe_file.md b/docs/reference/recipe_file.md index ea00bb255..330499e7d 100644 --- a/docs/reference/recipe_file.md +++ b/docs/reference/recipe_file.md @@ -420,6 +420,34 @@ source: - include/**/private.h ``` +The `filter` field is available for `path`, `url`, and `git` sources. It is +applied to the files that are copied into the work directory — the copied tree +for directory `path` sources, the contents of the extracted archive for `url` +sources (and `path` sources pointing to an archive), and the checked-out tree +for `git` sources. This is useful for trimming large sources down to the parts +you actually need to build: + +```yaml title="recipe.yaml" +source: + - url: https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-${{ version }}.tar.gz + sha256: ad18b70e287954c3d62bc7e0b86e7b7af2adf87bcfce21c15fe717f101d7aace + filter: + # we don't want to accidentally build more than clang here + - cmake/* + - clang/* + - clang-tools-extra/* +``` + +```yaml title="recipe.yaml" +source: + url: https://github.com/pytorch/pytorch/releases/download/v${{ version }}/pytorch-v${{ version }}.tar.gz + sha256: 757145cfd55c7c8c01f58c959f76230641cc67fdd1d8b6a130f93ad1bc116f5f + filter: + exclude: + # ensure we use our own fmt + - third_party/fmt/* +``` + ## Build section Specifies build information. diff --git a/test-data/recipes/source_filter_archive/archive.tar.gz b/test-data/recipes/source_filter_archive/archive.tar.gz new file mode 100644 index 000000000..2e4d245d7 Binary files /dev/null and b/test-data/recipes/source_filter_archive/archive.tar.gz differ diff --git a/test-data/recipes/source_filter_archive/recipe.yaml b/test-data/recipes/source_filter_archive/recipe.yaml new file mode 100644 index 000000000..e892f027c --- /dev/null +++ b/test-data/recipes/source_filter_archive/recipe.yaml @@ -0,0 +1,22 @@ +package: + name: source_filter_archive + version: 1.0.0 + +source: + # Local tarball that gets extracted; the filter must be applied to the + # extracted contents (same code path as URL sources pointing to archives). + - path: ./archive.tar.gz + filter: + include: + - "cmake/*" + - "clang/*" + exclude: + - "third_party/*" + +build: + script: + - echo "running build (test)" + - test -f cmake/CMakeLists.txt + - test -f clang/main.cpp + - test ! -e third_party + - test ! -f README.md diff --git a/test/end-to-end/test_simple.py b/test/end-to-end/test_simple.py index 6288f03a6..05f8a2a89 100644 --- a/test/end-to-end/test_simple.py +++ b/test/end-to-end/test_simple.py @@ -1288,6 +1288,18 @@ def test_source_filter(rattler_build: RattlerBuild, recipes: Path, tmp_path: Pat rattler_build(*args) +def test_source_filter_archive( + rattler_build: RattlerBuild, recipes: Path, tmp_path: Path +): + # The filter is applied to the extracted contents of an archive source. + path_to_recipe = recipes / "source_filter_archive" + args = rattler_build.build_args( + path_to_recipe, + tmp_path, + ) + rattler_build(*args) + + def test_nushell_script_detection( rattler_build: RattlerBuild, recipes: Path, tmp_path: Path ):