From 019c2e9089dce7597dc27245bb32f707c5730dd4 Mon Sep 17 00:00:00 2001 From: Kyle Into Date: Fri, 30 Oct 2026 16:43:00 -0700 Subject: [PATCH 1/4] Add `--exclude-scripts` Summary: Stack context: let `uv check` run Pyrefly (astral-sh/uv#22941): check PEP 723 scripts in their own uv environments, and check each selected workspace member as a project. Example workspace: ``` pyproject.toml # root member `demo`; [tool.pyrefly] project-excludes = ["**/vendor"] src/demo/main.py scripts/tool.py # PEP 723 script: dependencies = ["six"] packages/a/m.py # member `a`, no Pyrefly config packages/a/vendor/m.py packages/b/pyproject.toml # member `b`, with its own [tool.pyrefly] packages/b/m.py ``` - `uv check --package demo` runs `pyrefly check --exclude-scripts --extend-project-excludes=packages/a --extend-project-excludes=packages/b --project-dir=.`: it checks `src/demo/main.py`. - `uv check --package a` runs `pyrefly check --exclude-scripts --project-dir=packages/a`: it checks `packages/a/m.py`, not `packages/a/vendor/m.py`. - `uv check --all-packages` runs `pyrefly check --exclude-scripts --project-dir=. --project-dir=packages/a --project-dir=packages/b`: it checks each member with the config that governs it. - `uv check --script scripts/tool.py` runs `pyrefly check --script-command= --script-command=workspace ... --script-command=--script -- scripts/tool.py`: it resolves `six` from the script's own environment. - Add `pyrefly check --exclude-scripts`. Pyrefly skips the files with a PEP 723 `# /// script` block that it finds in the project or in the given directories, and still checks files given explicitly. - Pyrefly finds a script by its `# /// script` metadata block, which `script::metadata` parses. - `uv check` checks a project without its scripts, which must be checked with `uv check --script`. ty finds the scripts in its own walk and skips them with the same flag, so uv only passes the flag. Without it, uv must find the scripts itself and pass each one as an exclude. Example: `pyrefly check --exclude-scripts` checks `src/demo/main.py` and `packages/*/m.py`, and skips `scripts/tool.py`. `pyrefly check --exclude-scripts -- . scripts/tool.py` still checks `scripts/tool.py`, because it is given explicitly. Reviewed By: rchen152 Differential Revision: D128114982 --- crates/pyrefly_config/src/environment.rs | 1 + .../pyrefly_config/src/environment/script.rs | 47 +++++++ crates/pyrefly_util/src/globs.rs | 9 ++ pyrefly/lib/commands/files.rs | 117 ++++++++++++++++-- website/docs/configuration.mdx | 5 + 5 files changed, 172 insertions(+), 7 deletions(-) create mode 100644 crates/pyrefly_config/src/environment/script.rs diff --git a/crates/pyrefly_config/src/environment.rs b/crates/pyrefly_config/src/environment.rs index 1e15f2e5cc..36d5360047 100644 --- a/crates/pyrefly_config/src/environment.rs +++ b/crates/pyrefly_config/src/environment.rs @@ -16,4 +16,5 @@ pub mod environment; pub(crate) mod interpreter_query; pub mod interpreters; pub(crate) mod pixi; +pub mod script; pub(crate) mod venv; diff --git a/crates/pyrefly_config/src/environment/script.rs b/crates/pyrefly_config/src/environment/script.rs new file mode 100644 index 0000000000..8fd80f9f40 --- /dev/null +++ b/crates/pyrefly_config/src/environment/script.rs @@ -0,0 +1,47 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! PEP 723 scripts, which carry their own dependency metadata. + +/// Returns the contents of the PEP 723 `script` metadata block in `source`, if it has one. +pub fn metadata(source: &str) -> Option<&str> { + let mut start = None; + let mut offset = 0; + for line in source.split_inclusive('\n') { + let text = line.trim_end_matches(['\n', '\r']); + match start { + None if text == "# /// script" => start = Some(offset + line.len()), + Some(start) if text == "# ///" => return Some(&source[start..offset]), + Some(_) if text != "#" && !text.starts_with("# ") => start = None, + _ => {} + } + offset += line.len(); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_metadata() { + assert_eq!( + metadata( + "#!/usr/bin/env python\n# /// script\n# dependencies = [\"six\"]\n#\n# ///\nimport six\n" + ), + Some("# dependencies = [\"six\"]\n#\n"), + ); + assert_eq!( + metadata("# /// script\r\n# requires-python = \">=3.12\"\r\n# ///\r\n"), + Some("# requires-python = \">=3.12\"\r\n"), + ); + assert_eq!(metadata("# /// script\nimport six\n# ///\n"), None); + assert_eq!(metadata("# /// pyproject\n# ///\n"), None); + assert_eq!(metadata("x = 1\n"), None); + } +} diff --git a/crates/pyrefly_util/src/globs.rs b/crates/pyrefly_util/src/globs.rs index 4f85fd4574..aefde82926 100644 --- a/crates/pyrefly_util/src/globs.rs +++ b/crates/pyrefly_util/src/globs.rs @@ -409,6 +409,15 @@ impl Globs { res } + /// The existing files named by patterns without wildcards. [`Globs::files_iter`] yields these + /// paths as they are, without walking for them. + pub fn explicit_files(&self) -> impl Iterator { + self.0 + .iter() + .filter(|pattern| pattern.is_explicit_file_pattern()) + .map(Glob::as_path) + } + /// Returns true if the given file matches any of the contained globs. /// Directory-style matching is handled by each [`Glob`]'s precomputed /// `/**` variant. diff --git a/pyrefly/lib/commands/files.rs b/pyrefly/lib/commands/files.rs index d05fba4bcd..3658927a49 100644 --- a/pyrefly/lib/commands/files.rs +++ b/pyrefly/lib/commands/files.rs @@ -5,6 +5,7 @@ * LICENSE file in the root directory of this source tree. */ +use std::fs; use std::path::Path; use std::path::PathBuf; @@ -12,6 +13,7 @@ use anyhow::Context as _; use clap::Parser; use dupe::Dupe; use pyrefly_config::args::ConfigOverrideArgs; +use pyrefly_config::environment::script; use pyrefly_config::resolve_unconfigured::UnconfiguredOverride; use pyrefly_util::absolutize::Absolutize as _; use pyrefly_util::arc_id::ArcId; @@ -20,6 +22,7 @@ use pyrefly_util::globs::FilteredGlobs; use pyrefly_util::globs::Globs; use pyrefly_util::globs::HiddenDirFilter; use pyrefly_util::includes::Includes; +use starlark_map::small_set::SmallSet; use tracing::debug; use tracing::info; use tracing::warn; @@ -70,6 +73,10 @@ pub struct FilesArgs { /// Files to exclude when type checking. #[arg(long)] project_excludes: Option>, + /// Skip the PEP 723 scripts that Pyrefly finds in the project or in the given directories. + /// Files given explicitly are still checked. + #[arg(long)] + exclude_scripts: bool, /// Explicitly set the Pyrefly configuration to use when type checking. /// In "single-file checking mode," this config is applied to all files being checked, ignoring @@ -246,7 +253,7 @@ fn get_globs_and_config_for_project( } /// Get inputs for a per-file check. If an explicit config is passed in, we use it; otherwise, we -/// find configs via upward search from each file. +/// find configs via upward search from each file. `files_to_check` must be absolute. /// /// Also returns the `UpsellDecision`: explicit `--config` always means /// `Skip` (the loaded config is real and never carries a synthesized @@ -260,7 +267,6 @@ fn get_globs_and_config_for_files( args: ConfigOverrideArgs, wrapper: Option, ) -> anyhow::Result<(Box, ConfigFinder, UpsellDecision)> { - let files_to_check = absolutize(files_to_check); let args_disable_excludes_heuristics = args.disable_project_excludes_heuristics(); let get_project_excludes_and_heuristics = move |config: Option<&ConfigFile>| { let mut project_excludes = project_excludes.unwrap_or_default(); @@ -329,6 +335,46 @@ fn get_globs_and_config_for_files( Ok((Box::new(globs), config_finder, upsell)) } +/// Skips the PEP 723 scripts among the files of `inner`, except the `explicit_files` that the user +/// names, which `inner` yields as they are. A script runs in its own environment, so a tool such +/// as `uv check` checks scripts separately from the project. +struct ExcludeScripts { + inner: Box, + explicit_files: SmallSet, +} + +impl ExcludeScripts { + fn keeps(&self, path: &Path) -> bool { + self.explicit_files.contains(path) + || path.extension().is_none_or(|extension| extension != "py") + || fs::read_to_string(path).map_or(true, |source| script::metadata(&source).is_none()) + } +} + +impl Includes for ExcludeScripts { + fn roots(&self) -> Vec { + self.inner.roots() + } + + fn files_iter(&self) -> anyhow::Result + '_>> { + Ok(Box::new( + self.inner.files_iter()?.filter(|path| self.keeps(path)), + )) + } + + fn covers(&self, path: &Path) -> bool { + self.inner.covers(path) && self.keeps(path) + } + + fn covers_ignoring_excludes(&self, path: &Path) -> bool { + self.inner.covers_ignoring_excludes(path) + } + + fn errors(&mut self) -> Vec { + self.inner.errors() + } +} + impl FilesArgs { /// The file arguments as given on the command line. Empty means project-checking mode. pub fn files(&self) -> &[String] { @@ -355,24 +401,38 @@ impl FilesArgs { } else { None }; - if self.files.is_empty() { + let files_to_check = absolutize(Globs::new(self.files)?); + let explicit_files = files_to_check + .explicit_files() + .map(Path::to_path_buf) + .collect(); + let (includes, config_finder, upsell) = if files_to_check.is_empty() { get_globs_and_config_for_project( self.config, project_excludes, config_override, wrapper, scope, - ) + )? } else { // File mode bypasses the config's globs, so `scope` is intentionally unused. get_globs_and_config_for_files( self.config, - Globs::new(self.files)?, + files_to_check, project_excludes, config_override, wrapper, - ) - } + )? + }; + let includes = if self.exclude_scripts { + Box::new(ExcludeScripts { + inner: includes, + explicit_files, + }) + } else { + includes + }; + Ok((includes, config_finder, upsell)) } pub fn get( @@ -385,7 +445,50 @@ impl FilesArgs { files, config, project_excludes: None, + exclude_scripts: false, } .resolve(args, wrapper) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_exclude_scripts() { + let tempdir = tempfile::tempdir().unwrap(); + let root = tempdir.path(); + let script = root.join("script.py"); + let module = root.join("module.py"); + let stub = root.join("stub.pyi"); + let metadata = "# /// script\n# dependencies = [\"six\"]\n# ///\n"; + fs::write(&script, metadata).unwrap(); + fs::write(&module, "x = 1\n").unwrap(); + fs::write(&stub, metadata).unwrap(); + let files = |explicit_files: Vec| { + let includes = ExcludeScripts { + inner: Box::new(FilteredGlobs::new( + Globs::new(vec![root.to_string_lossy().into_owned()]).unwrap(), + Globs::empty(), + None, + HiddenDirFilter::Disabled, + )), + explicit_files: explicit_files.into_iter().collect(), + }; + let mut files = includes.files_iter().unwrap().collect::>(); + files.sort(); + files + }; + assert_eq!( + files(Vec::new()), + vec![module.clone(), stub.clone()], + "a script found in a directory is skipped, and a stub is never a script", + ); + assert_eq!( + files(vec![script.clone()]), + vec![module, script, stub], + "an explicitly given script is checked", + ); + } +} diff --git a/website/docs/configuration.mdx b/website/docs/configuration.mdx index 7e9a678518..0ea56bea00 100644 --- a/website/docs/configuration.mdx +++ b/website/docs/configuration.mdx @@ -233,6 +233,11 @@ or remove `project-excludes`. we recommend you pull and set up your dependency from GitHub, but you can achieve this on files in your `site-packages-path` by setting `site-packages-path = []` in your config. + - `--exclude-scripts` also skips the PEP 723 scripts (files with a `# /// script` + block) that Pyrefly finds in the project or in the given directories, and still + checks the files that you give explicitly. A script runs in its own environment, so + `uv check --type-checker pyrefly` passes `--exclude-scripts` and checks a script + with `uv check --script`. ### `disable-project-excludes-heuristics` From bb01dde47378e0af233c8c9064360004f1cc0ee1 Mon Sep 17 00:00:00 2001 From: Kyle Into Date: Fri, 30 Oct 2026 16:43:00 -0700 Subject: [PATCH 2/4] Add a test of `--project-excludes` in a workspace Summary: Stack context: let `uv check` run Pyrefly (astral-sh/uv#22941): check PEP 723 scripts in their own uv environments, and check each selected workspace member as a project. Example workspace: ``` pyproject.toml # root member `demo`; [tool.pyrefly] project-excludes = ["**/vendor"] src/demo/main.py scripts/tool.py # PEP 723 script: dependencies = ["six"] packages/a/m.py # member `a`, no Pyrefly config packages/a/vendor/m.py packages/b/pyproject.toml # member `b`, with its own [tool.pyrefly] packages/b/m.py ``` - `uv check --package demo` runs `pyrefly check --exclude-scripts --extend-project-excludes=packages/a --extend-project-excludes=packages/b --project-dir=.`: it checks `src/demo/main.py`. - `uv check --package a` runs `pyrefly check --exclude-scripts --project-dir=packages/a`: it checks `packages/a/m.py`, not `packages/a/vendor/m.py`. - `uv check --all-packages` runs `pyrefly check --exclude-scripts --project-dir=. --project-dir=packages/a --project-dir=packages/b`: it checks each member with the config that governs it. - `uv check --script scripts/tool.py` runs `pyrefly check --script-command= --script-command=workspace ... --script-command=--script -- scripts/tool.py`: it resolves `six` from the script's own environment. - What: a scrut test with a uv-style workspace. It shows that in project mode `--project-excludes` replaces the `project-excludes` of the config. - Why: `uv check` must exclude the workspace members that it does not select, and keep the user's own excludes. This test shows that `--project-excludes` cannot do that. The next diff adds `--extend-project-excludes` and tests it in this workspace. Example: `pyrefly check --project-excludes=packages/b` checks `packages/a/vendor/m.py`, because the flag replaces the root `project-excludes = ["**/vendor"]`. Differential Revision: D128428110 --- test/workspace.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/workspace.md diff --git a/test/workspace.md b/test/workspace.md new file mode 100644 index 0000000000..94ca4c7a17 --- /dev/null +++ b/test/workspace.md @@ -0,0 +1,30 @@ +# Tests for checking the projects of a workspace + +## Set up a workspace + +The root config excludes every `vendor` directory. `packages/a` has no Pyrefly +config, so the root config governs it. `packages/b` has its own config, which +does not exclude `vendor`. Each file has one error, so the output shows which +files are checked. + +```scrut +$ mkdir -p $TMPDIR/workspace/packages/a/vendor $TMPDIR/workspace/packages/b/vendor && \ +> cd $TMPDIR/workspace && \ +> printf '[tool.pyrefly]\nproject-excludes = ["**/vendor"]\n' > pyproject.toml && \ +> printf '[project]\nname = "a"\n' > packages/a/pyproject.toml && \ +> printf '[project]\nname = "b"\n\n[tool.pyrefly]\n' > packages/b/pyproject.toml && \ +> for d in . packages/a packages/a/vendor packages/b packages/b/vendor; do echo "x: int = ''" > $d/m.py; done +[0] +``` + +## `--project-excludes` replaces the excludes of the config + +The root config excludes `vendor`, but `packages/a/vendor/m.py` is checked. + +```scrut +$ cd $TMPDIR/workspace && $PYREFLY check --project-excludes=packages/b --python-version 3.13.0 --output-format=min-text +ERROR m.py:1:* (glob) +*packages/a/m.py:1:* (glob) +*packages/a/vendor/m.py:1:* (glob) +[1] +``` From 6acc408d6607ff0b516edd25bf8e5358a1bbf56c Mon Sep 17 00:00:00 2001 From: Kyle Into Date: Fri, 30 Oct 2026 16:43:00 -0700 Subject: [PATCH 3/4] Add `--extend-project-excludes` to add to the configured excludes Summary: Stack context: let `uv check` run Pyrefly (astral-sh/uv#22941): check PEP 723 scripts in their own uv environments, and check each selected workspace member as a project. Example workspace: ``` pyproject.toml # root member `demo`; [tool.pyrefly] project-excludes = ["**/vendor"] src/demo/main.py scripts/tool.py # PEP 723 script: dependencies = ["six"] packages/a/m.py # member `a`, no Pyrefly config packages/a/vendor/m.py packages/b/pyproject.toml # member `b`, with its own [tool.pyrefly] packages/b/m.py ``` - `uv check --package demo` runs `pyrefly check --exclude-scripts --extend-project-excludes=packages/a --extend-project-excludes=packages/b --project-dir=.`: it checks `src/demo/main.py`. - `uv check --package a` runs `pyrefly check --exclude-scripts --project-dir=packages/a`: it checks `packages/a/m.py`, not `packages/a/vendor/m.py`. - `uv check --all-packages` runs `pyrefly check --exclude-scripts --project-dir=. --project-dir=packages/a --project-dir=packages/b`: it checks each member with the config that governs it. - `uv check --script scripts/tool.py` runs `pyrefly check --script-command= --script-command=workspace ... --script-command=--script -- scripts/tool.py`: it resolves `six` from the script's own environment. - What: `--extend-project-excludes` adds patterns to the excludes that apply. In project mode, that is the `project-excludes` of the config (or `coverage.excludes` for `pyrefly coverage`). When files are given, that is `--project-excludes` plus the default excludes. `--project-excludes` still replaces the configured excludes. - Why: `uv check` must exclude the workspace members that it does not select, and keep the user's own excludes. `--project-excludes` drops them. A separate flag does not change the behavior of `--project-excludes` for existing users. Ruff has the same pair: `--exclude` and `--extend-exclude`. Example: `pyrefly check --extend-project-excludes=packages/b` checks `src/demo/main.py` and `packages/a/m.py`. It skips `packages/b`, and it skips `packages/a/vendor/m.py` because the root exclude still applies. Differential Revision: D128428167 --- crates/pyrefly_util/src/globs.rs | 5 +++++ pyrefly/lib/commands/files.rs | 12 ++++++++++++ test/workspace.md | 9 +++++++++ website/docs/configuration.mdx | 6 ++++-- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/crates/pyrefly_util/src/globs.rs b/crates/pyrefly_util/src/globs.rs index aefde82926..905d87cabc 100644 --- a/crates/pyrefly_util/src/globs.rs +++ b/crates/pyrefly_util/src/globs.rs @@ -1071,6 +1071,11 @@ impl FilteredGlobs { filter: GlobFilter::new(excludes, ignorefile_search_start, hidden_dir_filter), } } + + /// Adds `excludes` to the excludes of this [`FilteredGlobs`]. + pub fn add_excludes(&mut self, excludes: &[Glob]) { + self.filter.excludes.append(excludes); + } } impl Includes for FilteredGlobs { diff --git a/pyrefly/lib/commands/files.rs b/pyrefly/lib/commands/files.rs index 3658927a49..04ad3dd043 100644 --- a/pyrefly/lib/commands/files.rs +++ b/pyrefly/lib/commands/files.rs @@ -73,6 +73,10 @@ pub struct FilesArgs { /// Files to exclude when type checking. #[arg(long)] project_excludes: Option>, + /// Files to exclude when type checking, in addition to the `project-excludes` of the config + /// in project mode, or to `--project-excludes` and the default excludes when files are given. + #[arg(long)] + extend_project_excludes: Vec, /// Skip the PEP 723 scripts that Pyrefly finds in the project or in the given directories. /// Files given explicitly are still checked. #[arg(long)] @@ -185,6 +189,7 @@ pub fn get_config_finder_for_snippet( fn get_globs_and_config_for_project( config: Option, project_excludes: Option, + extend_project_excludes: &Globs, args: ConfigOverrideArgs, wrapper: Option, scope: ConfigScope, @@ -237,6 +242,7 @@ fn get_globs_and_config_for_project( debug!("Config is: {}", config); let mut filtered_globs = config.get_filtered_globs(project_excludes, scope); + filtered_globs.add_excludes(extend_project_excludes.globs()); filtered_globs .errors() .into_iter() @@ -264,12 +270,14 @@ fn get_globs_and_config_for_files( config: Option, files_to_check: Globs, project_excludes: Option, + extend_project_excludes: &Globs, args: ConfigOverrideArgs, wrapper: Option, ) -> anyhow::Result<(Box, ConfigFinder, UpsellDecision)> { let args_disable_excludes_heuristics = args.disable_project_excludes_heuristics(); let get_project_excludes_and_heuristics = move |config: Option<&ConfigFile>| { let mut project_excludes = project_excludes.unwrap_or_default(); + project_excludes.append(extend_project_excludes.globs()); let disable = args_disable_excludes_heuristics .or_else(|| Some(config?.disable_project_excludes_heuristics)) .unwrap_or(false); @@ -401,6 +409,7 @@ impl FilesArgs { } else { None }; + let extend_project_excludes = absolutize(Globs::new(self.extend_project_excludes)?); let files_to_check = absolutize(Globs::new(self.files)?); let explicit_files = files_to_check .explicit_files() @@ -410,6 +419,7 @@ impl FilesArgs { get_globs_and_config_for_project( self.config, project_excludes, + &extend_project_excludes, config_override, wrapper, scope, @@ -420,6 +430,7 @@ impl FilesArgs { self.config, files_to_check, project_excludes, + &extend_project_excludes, config_override, wrapper, )? @@ -445,6 +456,7 @@ impl FilesArgs { files, config, project_excludes: None, + extend_project_excludes: Vec::new(), exclude_scripts: false, } .resolve(args, wrapper) diff --git a/test/workspace.md b/test/workspace.md index 94ca4c7a17..0cae79ad1f 100644 --- a/test/workspace.md +++ b/test/workspace.md @@ -28,3 +28,12 @@ ERROR m.py:1:* (glob) *packages/a/vendor/m.py:1:* (glob) [1] ``` + +## `--extend-project-excludes` adds to the excludes of the config + +```scrut +$ cd $TMPDIR/workspace && $PYREFLY check --extend-project-excludes=packages/b --python-version 3.13.0 --output-format=min-text +ERROR m.py:1:* (glob) +*packages/a/m.py:1:* (glob) +[1] +``` diff --git a/website/docs/configuration.mdx b/website/docs/configuration.mdx index 0ea56bea00..ca1ed5a3d7 100644 --- a/website/docs/configuration.mdx +++ b/website/docs/configuration.mdx @@ -210,7 +210,8 @@ or remove `project-excludes`. - Default: `["**/node_modules", "**/__pycache__", "**/venv/**", "**/.[!/.]*/**"]` + anything in your `site-packages-path` (even from the interpreter) unless it would exclude items in your [`search-path`](#search-path). -- Flag equivalent: `--project-excludes` +- Flag equivalent: `--project-excludes`, which replaces `project-excludes`. + `--extend-project-excludes` adds patterns to `project-excludes` instead. - Equivalent configs: `exclude` in Pyright and mypy - Notes: - While not explicitly part of `project-excludes`, there are several patterns that are @@ -267,7 +268,8 @@ project setting when unset. - Type: table with `includes` and `excludes` keys, each a list of [filesystem glob patterns](#filesystem-globbing) - Default: `project-includes` / `project-excludes` -- Flag equivalent: none, though `--project-excludes` takes precedence over `coverage.excludes` +- Flag equivalent: none, though `--project-excludes` takes precedence over `coverage.excludes`, + and `--extend-project-excludes` adds to `coverage.excludes` - Notes: - Like the project globs, these are not consulted when passing in `FILES...`. - `coverage.excludes` gets the same default excludes appended as `project-excludes` does, From af7845e508f8cf9a9d923ffeffdc64143c622177 Mon Sep 17 00:00:00 2001 From: Kyle Into Date: Fri, 30 Oct 2026 16:43:00 -0700 Subject: [PATCH 4/4] Add `--project-dir` to check the project of a directory Summary: Stack context: let `uv check` run Pyrefly (astral-sh/uv#22941): check PEP 723 scripts in their own uv environments, and check each selected workspace member as a project. Example workspace: ``` pyproject.toml # root member `demo`; [tool.pyrefly] project-excludes = ["**/vendor"] src/demo/main.py scripts/tool.py # PEP 723 script: dependencies = ["six"] packages/a/m.py # member `a`, no Pyrefly config packages/a/vendor/m.py packages/b/pyproject.toml # member `b`, with its own [tool.pyrefly] packages/b/m.py ``` - `uv check --package demo` runs `pyrefly check --exclude-scripts --extend-project-excludes=packages/a --extend-project-excludes=packages/b --project-dir=.`: it checks `src/demo/main.py`. - `uv check --package a` runs `pyrefly check --exclude-scripts --project-dir=packages/a`: it checks `packages/a/m.py`, not `packages/a/vendor/m.py`. - `uv check --all-packages` runs `pyrefly check --exclude-scripts --project-dir=. --project-dir=packages/a --project-dir=packages/b`: it checks each member with the config that governs it. - `uv check --script scripts/tool.py` runs `pyrefly check --script-command= --script-command=workspace ... --script-command=--script -- scripts/tool.py`: it resolves `six` from the script's own environment. - What: `pyrefly check --project-dir ` runs project mode for a directory. It finds the config from ``, as `pyrefly check` does from the working directory, and checks the files that this config includes and that are under ``. The flag repeats, so one run checks several projects. A file belongs to the deepest given directory that contains it, and is checked with the config of that directory. - Why: `uv check` selects workspace members by their root directories. With `pyrefly check `, Pyrefly ignores the `project-excludes` of the config, so `uv check` and `pyrefly check` disagree. Path mode cannot apply the config excludes in a clear way when configs nest or when an exclude matches a path that the user names. Project mode already has one clear rule: one config selects the files and checks them. `--project-dir` keeps that rule and only limits it to a directory, so a member gets the same files and the same config as in `pyrefly check`. - Why it works: most uv members have no `[tool.pyrefly]`, so the config of a member is often the workspace root config. Each distinct config is walked once with its own includes and excludes, and a file is kept only by the config of the directory that it belongs to. The config finder gives each file under a directory that same config, and gives other files (dependencies) the config of the normal search. Example: `pyrefly check --project-dir=packages/a` checks `packages/a/m.py` with the root config, so `packages/a/vendor/m.py` stays excluded. `pyrefly check --project-dir=. --project-dir=packages/a --project-dir=packages/b` (what `uv check --all-packages` runs) checks `src/demo/main.py` and `packages/a/m.py` with the root config, and `packages/b/m.py` with the config of `packages/b`. Before, uv ran `pyrefly check packages/a`, which also checked `packages/a/vendor/m.py`. Differential Revision: D128428212 --- pyrefly/lib/commands/config_finder.rs | 42 +++++- pyrefly/lib/commands/files.rs | 181 ++++++++++++++++++++++++-- test/workspace.md | 56 ++++++++ website/docs/configuration.mdx | 7 + 4 files changed, 276 insertions(+), 10 deletions(-) diff --git a/pyrefly/lib/commands/config_finder.rs b/pyrefly/lib/commands/config_finder.rs index d76f7b6527..1cf49c8525 100644 --- a/pyrefly/lib/commands/config_finder.rs +++ b/pyrefly/lib/commands/config_finder.rs @@ -189,6 +189,21 @@ pub fn default_config_finder_with_overrides( ) } +/// A config finder for `pyrefly check --project-dir`. A Python file under one of `dirs` gets the +/// config of the first directory that contains it, so `dirs` must be ordered deepest first. Any +/// other file gets the config that [`default_config_finder_with_overrides`] finds. +pub fn project_dirs_config_finder( + dirs: Vec<(PathBuf, ArcId)>, + args: ConfigOverrideArgs, + wrapper: Option, +) -> ConfigFinder { + config_finder_with_dirs( + Arc::new(DefaultConfigConfigurerWithOverrides::new(args, false)), + wrapper, + dirs, + ) +} + /// Create a standard `ConfigFinder`, using the provided [`ConfigConfigurer`] to finalize /// the config before caching/returning it. /// @@ -197,6 +212,16 @@ pub fn default_config_finder_with_overrides( pub fn standard_config_finder( configure: Arc, wrapper: Option, +) -> ConfigFinder { + config_finder_with_dirs(configure, wrapper, Vec::new()) +} + +/// [`standard_config_finder`], except that a Python file under one of `dirs` gets the config of +/// the first directory that contains it. +fn config_finder_with_dirs( + configure: Arc, + wrapper: Option, + dirs: Vec<(PathBuf, ArcId)>, ) -> ConfigFinder { let configure = match wrapper { Some(wrap) => wrap(configure), @@ -254,10 +279,23 @@ pub fn standard_config_finder( ConfigFinder::new_custom( Box::new(move |_, path| { - Ok(GENERATED_FILE_CONFIG_OVERRIDE + if let Some(config) = GENERATED_FILE_CONFIG_OVERRIDE .read() .get(&path.module_path_buf()) - .cloned()) + { + return Ok(Some(config.dupe())); + } + let file = match path.details() { + ModulePathDetails::FileSystem(file) | ModulePathDetails::Memory(file) => file, + ModulePathDetails::Namespace(_) + | ModulePathDetails::BundledTypeshed(_) + | ModulePathDetails::BundledTypeshedThirdParty(_) + | ModulePathDetails::BundledThirdParty(_) => return Ok(None), + }; + Ok(dirs + .iter() + .find(|(dir, _)| file.starts_with(dir)) + .map(|(_, config)| config.dupe())) }), Box::new(move |file| { let (file_config, parse_errors) = ConfigFile::from_file(file); diff --git a/pyrefly/lib/commands/files.rs b/pyrefly/lib/commands/files.rs index 04ad3dd043..f6be151147 100644 --- a/pyrefly/lib/commands/files.rs +++ b/pyrefly/lib/commands/files.rs @@ -30,6 +30,7 @@ use tracing::warn; use crate::commands::config_finder::ConfigConfigurerWrapper; use crate::commands::config_finder::apply_unconfigured_resolver_if_applicable; use crate::commands::config_finder::default_config_finder_with_overrides; +use crate::commands::config_finder::project_dirs_config_finder; use crate::config::config::ConfigFile; use crate::config::config::ConfigScope; use crate::config::config::ConfigSource; @@ -81,6 +82,12 @@ pub struct FilesArgs { /// Files given explicitly are still checked. #[arg(long)] exclude_scripts: bool, + /// Check the project of this directory, limited to the files under it. The directory uses the + /// config that `pyrefly check` finds from it, with the `project-includes` and + /// `project-excludes` of that config. Repeat the flag to check several directories in one run. + /// A file belongs to the deepest given directory that contains it. + #[arg(long, value_name = "DIR", conflicts_with = "files")] + project_dir: Vec, /// Explicitly set the Pyrefly configuration to use when type checking. /// In "single-file checking mode," this config is applied to all files being checked, ignoring @@ -135,7 +142,18 @@ pub fn get_project_config_for_current_dir( ) -> anyhow::Result<(ArcId, Vec)> { let current_dir = std::env::current_dir().context("cannot identify current dir")?; let config_finder = default_config_finder_with_overrides(args.clone(), false, wrapper); - let config = config_finder.directory(¤t_dir).unwrap_or_else(|| { + let config = get_project_config_for_dir(&config_finder, ¤t_dir, &args); + Ok((config, config_finder.errors())) +} + +/// The config that `config_finder` finds for `dir`, or a config synthesized at `dir` if there is +/// none. +fn get_project_config_for_dir( + config_finder: &ConfigFinder, + dir: &Path, + args: &ConfigOverrideArgs, +) -> ArcId { + config_finder.directory(dir).unwrap_or_else(|| { // No marker found upward. Run the synthesized config through the // same unconfigured resolver wiring that the file-mode path uses // (via `DefaultConfigConfigurerWithOverrides`); without this, @@ -144,19 +162,17 @@ pub fn get_project_config_for_current_dir( // file-mode upsell would never fire for project-mode checks, and // a nearby mypy.ini / pyrightconfig.json would be migrated for // file mode but ignored here. - let mut synthesized = - ConfigFile::init_at_root(¤t_dir, &ProjectLayout::new(¤t_dir), false); + let mut synthesized = ConfigFile::init_at_root(dir, &ProjectLayout::new(dir), false); apply_unconfigured_resolver_if_applicable( &mut synthesized, - Some(¤t_dir), + Some(dir), UnconfiguredOverride::Auto, ); - let (config, errors) = args.override_config_at(synthesized, Some(¤t_dir)); + let (config, errors) = args.override_config_at(synthesized, Some(dir)); // Since this is a config we generated, these are likely internal errors. debug_log(errors); config - }); - Ok((config, config_finder.errors())) + }) } pub fn get_config_finder_for_snippet( @@ -258,6 +274,144 @@ fn get_globs_and_config_for_project( Ok((Box::new(filtered_globs), config_finder, upsell)) } +/// Get inputs for `--project-dir`. Each directory gets the files and the config that +/// `pyrefly check` gives it, limited to the files under the directory. +fn get_globs_and_config_for_project_dirs( + config: Option, + project_dirs: Vec, + project_excludes: Option, + extend_project_excludes: &Globs, + args: ConfigOverrideArgs, + wrapper: Option, + scope: ConfigScope, +) -> anyhow::Result<(Box, ConfigFinder, UpsellDecision)> { + let mut errors = Vec::new(); + let explicit_config = config.map(|path| { + let (config, explicit_errors) = get_explicit_config(&path, args.clone()); + errors.extend(explicit_errors); + config + }); + let finder = default_config_finder_with_overrides(args.clone(), false, wrapper.clone()); + let mut dirs = Vec::new(); + for dir in project_dirs { + let dir = dir.absolutize(); + if !dir.is_dir() { + return Err(anyhow::anyhow!( + "`--project-dir` path `{}` is not a directory", + dir.display() + )); + } + let config = explicit_config + .dupe() + .unwrap_or_else(|| get_project_config_for_dir(&finder, &dir, &args)); + if config.build_system.is_some() { + return Err(anyhow::anyhow!( + "Cannot run build system in project mode, you must provide files to check" + )); + } + debug!( + "Project directory `{}` has config: {}", + dir.display(), + config + ); + dirs.push((dir, config)); + } + errors.extend(finder.errors()); + // A path sorts after its ancestors, so the reverse order puts the deepest directory first. + dirs.sort_by(|(a, _), (b, _)| b.cmp(a)); + dirs.dedup_by(|(a, _), (b, _)| a == b); + + let mut includes: Vec<(ArcId, FilteredGlobs)> = Vec::new(); + for (_, config) in &dirs { + if !includes.iter().any(|(seen, _)| seen == config) { + let mut globs = config.get_filtered_globs(project_excludes.clone(), scope); + globs.add_excludes(extend_project_excludes.globs()); + errors.extend(globs.errors().into_iter().map(ConfigError::warn)); + includes.push((config.dupe(), globs)); + } + } + let upsell = match dirs + .iter() + .find_map(|(_, config)| config.synthesized_preset_reason) + { + Some(reason) => UpsellDecision::Show(reason), + None => UpsellDecision::Skip, + }; + let config_finder = project_dirs_config_finder(dirs.clone(), args, wrapper); + add_config_errors(&config_finder, errors)?; + Ok(( + Box::new(ProjectDirs { dirs, includes }), + config_finder, + upsell, + )) +} + +/// The files of `--project-dir`. A file belongs to the deepest given directory that contains it, +/// and is checked if the config of that directory includes it. +struct ProjectDirs { + /// Each given directory and its config, deepest first. + dirs: Vec<(PathBuf, ArcId)>, + /// The includes of each distinct config in `dirs`. + includes: Vec<(ArcId, FilteredGlobs)>, +} + +impl ProjectDirs { + /// The config of the directory that `path` belongs to. + fn config(&self, path: &Path) -> Option<&ArcId> { + self.dirs + .iter() + .find(|(dir, _)| path.starts_with(dir)) + .map(|(_, config)| config) + } + + /// The includes of the config of the directory that `path` belongs to. + fn includes(&self, path: &Path) -> Option<&FilteredGlobs> { + let config = self.config(path)?; + self.includes + .iter() + .find(|(seen, _)| seen == config) + .map(|(_, includes)| includes) + } +} + +impl Includes for ProjectDirs { + fn roots(&self) -> Vec { + self.dirs.iter().map(|(dir, _)| dir.clone()).collect() + } + + fn files_iter(&self) -> anyhow::Result + '_>> { + // Each config is walked once. A file is kept only by the config of its directory, so no + // file is returned twice. + let mut files = Vec::new(); + for (config, includes) in &self.includes { + files.extend( + includes + .files_iter()? + .filter(|path| self.config(path) == Some(config)), + ); + } + files.sort(); + Ok(Box::new(files.into_iter())) + } + + fn covers(&self, path: &Path) -> bool { + self.includes(path) + .is_some_and(|includes| includes.covers(path)) + } + + fn covers_ignoring_excludes(&self, path: &Path) -> bool { + self.includes(path) + .is_some_and(|includes| includes.covers_ignoring_excludes(path)) + } + + fn errors(&mut self) -> Vec { + self.includes + .iter_mut() + .flat_map(|(_, includes)| includes.errors()) + .collect() + } +} + /// Get inputs for a per-file check. If an explicit config is passed in, we use it; otherwise, we /// find configs via upward search from each file. `files_to_check` must be absolute. /// @@ -415,7 +569,17 @@ impl FilesArgs { .explicit_files() .map(Path::to_path_buf) .collect(); - let (includes, config_finder, upsell) = if files_to_check.is_empty() { + let (includes, config_finder, upsell) = if !self.project_dir.is_empty() { + get_globs_and_config_for_project_dirs( + self.config, + self.project_dir, + project_excludes, + &extend_project_excludes, + config_override, + wrapper, + scope, + )? + } else if files_to_check.is_empty() { get_globs_and_config_for_project( self.config, project_excludes, @@ -458,6 +622,7 @@ impl FilesArgs { project_excludes: None, extend_project_excludes: Vec::new(), exclude_scripts: false, + project_dir: Vec::new(), } .resolve(args, wrapper) } diff --git a/test/workspace.md b/test/workspace.md index 0cae79ad1f..b758227e72 100644 --- a/test/workspace.md +++ b/test/workspace.md @@ -37,3 +37,59 @@ ERROR m.py:1:* (glob) *packages/a/m.py:1:* (glob) [1] ``` + +## `--project-dir` checks the files of the project under a directory + +The root config governs `packages/a`, so its `vendor` directory is excluded. + +```scrut +$ cd $TMPDIR/workspace && $PYREFLY check --project-dir=packages/a --python-version 3.13.0 --output-format=min-text +*packages/a/m.py:1:* (glob) +[1] +``` + +## `--project-dir` uses the config of a directory that has one + +The config of `packages/b` does not exclude `vendor`. + +```scrut +$ cd $TMPDIR/workspace && $PYREFLY check --project-dir=packages/b --python-version 3.13.0 --output-format=min-text +*packages/b/m.py:1:* (glob) +*packages/b/vendor/m.py:1:* (glob) +[1] +``` + +## `--project-dir` checks the same files as `pyrefly check` + +`packages/b` is below `.`, so the root config governs it, the same as in +`pyrefly check`. + +```scrut +$ cd $TMPDIR/workspace && $PYREFLY check --project-dir=. --python-version 3.13.0 --output-format=min-text +ERROR m.py:1:* (glob) +*packages/a/m.py:1:* (glob) +*packages/b/m.py:1:* (glob) +[1] +``` + +```scrut +$ cd $TMPDIR/workspace && $PYREFLY check --python-version 3.13.0 --output-format=min-text +ERROR m.py:1:* (glob) +*packages/a/m.py:1:* (glob) +*packages/b/m.py:1:* (glob) +[1] +``` + +## A file belongs to the deepest `--project-dir` that contains it + +`packages/b` is given, so its files use its own config. Errors are sorted by +module name, and `packages/b/m.py` is the module `m` of that config. + +```scrut +$ cd $TMPDIR/workspace && $PYREFLY check --project-dir=. --project-dir=packages/b --python-version 3.13.0 --output-format=min-text +ERROR m.py:1:* (glob) +*packages/b/m.py:1:* (glob) +*packages/a/m.py:1:* (glob) +*packages/b/vendor/m.py:1:* (glob) +[1] +``` diff --git a/website/docs/configuration.mdx b/website/docs/configuration.mdx index ca1ed5a3d7..6fdd83fd27 100644 --- a/website/docs/configuration.mdx +++ b/website/docs/configuration.mdx @@ -122,6 +122,13 @@ correspond to different but useful ways we expect most people to interact with P file listed. `project-includes` and `project-excludes` are ignored from the config file, but it is used for all remaining config options. +`pyrefly check --project-dir ` runs project mode for a directory: it finds the config +from `` instead of the current working directory, and checks only the files of that +project that are under ``. Repeat the flag to check several projects in one run, such as +the members of a uv workspace. A file belongs to the deepest given directory that contains it, +and is checked with the config of that directory. `uv check --type-checker pyrefly` passes the +root directory of each workspace member that it checks. + ## Configuration Finding In both project checking mode and single-file checking mode (see [Type Checking Modes](#type-checking-modes)