diff --git a/packages/pre_commit_excludes/README.md b/packages/pre_commit_excludes/README.md index 75ecf9a..2794a94 100644 --- a/packages/pre_commit_excludes/README.md +++ b/packages/pre_commit_excludes/README.md @@ -4,6 +4,8 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pre-commit-excludes)](https://pypi.org/project/pre-commit-excludes/) [![PyPI - License](https://img.shields.io/pypi/l/pre-commit-excludes)](https://pypi.org/project/pre-commit-excludes/) +## Remove Unnecessary Excludes + `remove-unnecessary-excludes` finds lines in your exclude list that are no longer required and removes them from the supplied `.pre-commit-config.yaml`. The tool checks each exclude by running the affected hook without excludes and restores changes made by a failing hook when the exclude is still required. @@ -12,8 +14,6 @@ The tool checks each exclude by running the affected hook without excludes and r > Each path must be on its own line in a multiline YAML block scalar; compact inline exclude patterns are not rewritten. > Keeping exclusions as plain `|`-separated paths also makes it easier for humans to maintain an overview of what is excluded. -## Usage - ```shell # Cleanup the entire config for all hooks uvx --from pre-commit-excludes remove-unnecessary-excludes .pre-commit-config.yaml --pre-commit-binary prek --git-binary git --all @@ -24,3 +24,38 @@ uvx --from pre-commit-excludes remove-unnecessary-excludes .pre-commit-config.ya # Skip excludes that you want to keep uvx --from pre-commit-excludes remove-unnecessary-excludes .pre-commit-config.yaml --pre-commit-binary prek --git-binary git --all --skip-exclude "check-json:tests/invalid.json" ``` + +## Restrict Folder Excludes + +`restrict-folder-excludes` restricts excluded folders in the `.pre-commit-config.yaml` exclude list from a very broad folder exclude to more fine granular excludes. + +```shell +# Restrict all excluded folders for all hooks +uvx --from pre-commit-excludes restrict-folder-excludes .pre-commit-config.yaml --all + +# Restrict all excluded folders for specific hooks such as typos and ruff-check and skip the tests folder for check-json +uvx --from pre-commit-excludes restrict-folder-excludes .pre-commit-config.yaml --hook typos --hook ruff-check --skip-exclude "check-json:tests" +``` + +Example for the command above + +```yaml +# Before +exclude: | + (?x)^( + foo| + bar + ) + +# After +exclude: | + (?x)^( + foo/a| + foo/b| + foo/c| + foo/foo.txt| + foo/README.md| + bar/a| + bar/bar.txt + ) +``` diff --git a/packages/pre_commit_excludes/pre_commit_excludes/hook_utils.py b/packages/pre_commit_excludes/pre_commit_excludes/hook_utils.py index ab850d3..c2e6298 100644 --- a/packages/pre_commit_excludes/pre_commit_excludes/hook_utils.py +++ b/packages/pre_commit_excludes/pre_commit_excludes/hook_utils.py @@ -9,10 +9,11 @@ from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap +from ruamel.yaml.scalarstring import LiteralScalarString from ruamel.yaml.util import load_yaml_guess_indent if TYPE_CHECKING: - from collections.abc import Iterator, Mapping + from collections.abc import Callable, Iterator, Mapping, MutableMapping @dataclass(frozen=True) @@ -128,11 +129,11 @@ def get_hooks_to_cleanup(hooks: list[Hook], selected_hooks: list[str] | None) -> return [hook for hook in hooks if hook.id in selected_hooks] -def get_relative_excludes_by_hook(hooks: list[Hook], root_directory: Path) -> dict[str, set[str]]: +def get_relative_excludes_by_hook(hooks: list[Hook], root_directory: Path) -> dict[str, list[str]]: """Return hook excludes as config-relative POSIX paths grouped by hook ID.""" - relative_excludes: dict[str, set[str]] = {} + relative_excludes: dict[str, list[str]] = {} for hook in hooks: - relative_excludes.setdefault(hook.id, set()).update( + relative_excludes.setdefault(hook.id, []).extend( exclude.relative_to(root_directory).as_posix() for exclude in hook.exclude_paths ) return relative_excludes @@ -156,3 +157,20 @@ def load_round_trip_config(config_file: Path) -> tuple[CommentedMap, YAML]: if indent is not None: yaml.indent(sequence=indent, offset=block_sequence_indent) return (config if isinstance(config, CommentedMap) else CommentedMap()), yaml + + +def update_config_hook_excludes( + config_file: Path, + update_exclude: Callable[[MutableMapping[str, Any]], str | None], +) -> None: + """Apply exclude replacements to hooks and write the config if it changed.""" + config, yaml = load_round_trip_config(config_file) + changed = False + for hook_config in get_hook_configs_from_all_repos(config): + updated_exclude = update_exclude(hook_config) + if updated_exclude is not None and updated_exclude != hook_config.get("exclude"): + hook_config["exclude"] = LiteralScalarString(updated_exclude) + changed = True + + if changed: + yaml.dump(config, config_file) diff --git a/packages/pre_commit_excludes/pre_commit_excludes/remove_unnecessary_excludes.py b/packages/pre_commit_excludes/pre_commit_excludes/remove_unnecessary_excludes.py index 45eacdc..d8e3722 100644 --- a/packages/pre_commit_excludes/pre_commit_excludes/remove_unnecessary_excludes.py +++ b/packages/pre_commit_excludes/pre_commit_excludes/remove_unnecessary_excludes.py @@ -4,22 +4,22 @@ import re import subprocess import sys +from collections.abc import MutableMapping from dataclasses import dataclass from pathlib import Path +from typing import Any -from ruamel.yaml.comments import CommentedMap from ruamel.yaml.scalarstring import LiteralScalarString from pre_commit_excludes.args import SkippedExclude, create_default_parser from pre_commit_excludes.hook_utils import ( Hook, - get_hook_configs_from_all_repos, get_hooks_to_cleanup, get_relative_excludes_by_hook, get_skipped_excludes_relative_to_config, load_config, load_hooks, - load_round_trip_config, + update_config_hook_excludes, write_config, ) @@ -56,7 +56,7 @@ def run_pre_commit( ).returncode -def undo_changes(exclude_path: Path, git_binary) -> None: +def undo_changes(exclude_path: Path, git_binary: Path) -> None: subprocess.run([git_binary, "restore", exclude_path], check=True) @@ -122,29 +122,20 @@ def _remove_excludes_from_block(block: str, excludes: set[str]) -> str: return "\n".join(retained_lines) + trailing_newline -def _remove_excludes_from_hooks(config: CommentedMap, excludes_by_hook: dict[str, set[str]]) -> bool: - changed = False - hooks_to_update = [ - hook - for hook in get_hook_configs_from_all_repos(config) - if hook.get("id") in excludes_by_hook and isinstance(hook.get("exclude"), LiteralScalarString) - ] - for hook in hooks_to_update: - hook_id = hook["id"] - exclude = hook["exclude"] - updated_exclude = _remove_excludes_from_block(exclude, excludes_by_hook[hook_id]) - if updated_exclude != exclude: - hook["exclude"] = LiteralScalarString(updated_exclude) - changed = True - return changed - - def remove_excludes_from_config(config_file: Path, hooks_to_update: list[Hook]) -> None: """Remove matching exclude lines from hooks in a pre-commit config.""" - config, yaml = load_round_trip_config(config_file) - relative_excludes = get_relative_excludes_by_hook(hooks_to_update, config_file.parent) - if _remove_excludes_from_hooks(config, relative_excludes): - yaml.dump(config, config_file) + relative_excludes = { + hook_id: set(excludes) + for hook_id, excludes in get_relative_excludes_by_hook(hooks_to_update, config_file.parent).items() + } + + def remove_excludes_from_hook(hook_config: MutableMapping[str, Any]) -> str | None: + hook_id = hook_config.get("id") + if hook_id not in relative_excludes or not isinstance(hook_config.get("exclude"), LiteralScalarString): + return None + return _remove_excludes_from_block(hook_config["exclude"], relative_excludes[hook_id]) + + update_config_hook_excludes(config_file, remove_excludes_from_hook) def parse_arguments() -> argparse.Namespace: diff --git a/packages/pre_commit_excludes/pre_commit_excludes/restrict_folder_excludes.py b/packages/pre_commit_excludes/pre_commit_excludes/restrict_folder_excludes.py new file mode 100644 index 0000000..3f29621 --- /dev/null +++ b/packages/pre_commit_excludes/pre_commit_excludes/restrict_folder_excludes.py @@ -0,0 +1,80 @@ +"""Restrict folder excludes from a .pre-commit-config.yaml.""" + +import argparse +import sys +from collections.abc import Iterator, MutableMapping +from pathlib import Path +from typing import Any + +from pre_commit_excludes.args import create_default_parser +from pre_commit_excludes.hook_utils import ( + Hook, + SkippedExclude, + get_hooks_to_cleanup, + get_relative_excludes_by_hook, + get_skipped_excludes_relative_to_config, + load_hooks, + update_config_hook_excludes, +) + + +def _iter_restricted_excludes(hook: Hook, skipped_excludes: set[SkippedExclude]) -> Iterator[Path]: + for exclude in hook.exclude_paths: + if SkippedExclude(hook.id, exclude) in skipped_excludes or exclude.is_file(): + yield exclude + else: + yield from exclude.glob("*") + + +def restrict_folder_excludes(hooks_to_restrict: list[Hook], skipped_excludes: set[SkippedExclude]) -> list[Hook]: + """Replace directory excludes with excludes for each direct child in the directory.""" + return [Hook(hook.id, list(_iter_restricted_excludes(hook, skipped_excludes))) for hook in hooks_to_restrict] + + +def update_config_with_stricter_excludes(config_file: Path, restricted_hooks: list[Hook]) -> None: + """Update the exclude blocks for the restricted hooks in a pre-commit config.""" + excludes_by_hook = get_relative_excludes_by_hook(restricted_hooks, config_file.parent) + + def update_exclude(hook_config: MutableMapping[str, Any]) -> str | None: + hook_id = hook_config.get("id") + if hook_id not in excludes_by_hook: + return None + + excludes = excludes_by_hook[hook_id] + return ( + "\n".join( + [ + "(?x)^(", + *( + f" {exclude}{'|' if index < len(excludes) - 1 else ''}" + for index, exclude in enumerate(excludes) + ), + ")", + ] + ) + + "\n" + ) + + update_config_hook_excludes(config_file, update_exclude) + + +def parse_arguments() -> argparse.Namespace: + parser = create_default_parser(__doc__) + return parser.parse_args() + + +def main() -> int: + args = parse_arguments() + + hooks_with_excludes = load_hooks(args.config.parent, args.config) + hooks_to_restrict = hooks_with_excludes if args.all else get_hooks_to_cleanup(hooks_with_excludes, args.hook) + skipped_excludes = get_skipped_excludes_relative_to_config(args.skip_exclude, args.config.parent) + + restricted_hooks = restrict_folder_excludes(hooks_to_restrict, skipped_excludes) + update_config_with_stricter_excludes(args.config, restricted_hooks) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/pre_commit_excludes/pyproject.toml b/packages/pre_commit_excludes/pyproject.toml index 2b69b10..bd3793c 100644 --- a/packages/pre_commit_excludes/pyproject.toml +++ b/packages/pre_commit_excludes/pyproject.toml @@ -34,6 +34,7 @@ classifiers = [ [project.scripts] remove-unnecessary-excludes = "pre_commit_excludes.remove_unnecessary_excludes:main" +restrict-folder-excludes = "pre_commit_excludes.restrict_folder_excludes:main" [project.urls] Homepage = "https://github.com/hofbi/dev-tools" diff --git a/tests/pre_commit_excludes/test_hook_utils.py b/tests/pre_commit_excludes/test_hook_utils.py index 43f1eee..b9ed89c 100644 --- a/tests/pre_commit_excludes/test_hook_utils.py +++ b/tests/pre_commit_excludes/test_hook_utils.py @@ -156,7 +156,7 @@ def test_find_non_existing_paths_for_existing_files_should_return_empty_list(fs: assert hook_instance.find_non_existing_paths() == [] -def test_get_relative_excludes_by_hook_should_group_posix_paths_by_hook_id() -> None: +def test_get_relative_excludes_by_hook_should_group_ordered_posix_paths_by_hook_id() -> None: hooks = [ Hook("ruff", [Path("Repo/generated/foo.py")]), Hook("ruff", [Path("Repo/generated/bar.py")]), @@ -164,8 +164,8 @@ def test_get_relative_excludes_by_hook_should_group_posix_paths_by_hook_id() -> ] assert get_relative_excludes_by_hook(hooks, Path("Repo")) == { - "ruff": {"generated/foo.py", "generated/bar.py"}, - "black": {"generated/foo.py"}, + "ruff": ["generated/foo.py", "generated/bar.py"], + "black": ["generated/foo.py"], } diff --git a/tests/pre_commit_excludes/test_restrict_folder_excludes.py b/tests/pre_commit_excludes/test_restrict_folder_excludes.py new file mode 100644 index 0000000..08aeca2 --- /dev/null +++ b/tests/pre_commit_excludes/test_restrict_folder_excludes.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from pre_commit_excludes.hook_utils import Hook, SkippedExclude +from pre_commit_excludes.restrict_folder_excludes import restrict_folder_excludes, update_config_with_stricter_excludes + +if TYPE_CHECKING: + from pyfakefs.fake_filesystem import FakeFilesystem + + +def test_restrict_folder_excludes_for_folder_should_replace_folder_with_direct_children(fs: FakeFilesystem) -> None: + exclude_folder = Path("Repo/excluded") + first_file = exclude_folder / "foo.py" + nested_folder = exclude_folder / "nested" + second_file = nested_folder / "bar.py" + fs.create_file(first_file) + fs.create_file(second_file) + + hooks = [Hook("ruff", [exclude_folder])] + restricted_hooks = restrict_folder_excludes(hooks, set()) + + assert len(restricted_hooks) == 1 + assert restricted_hooks[0].id == "ruff" + assert restricted_hooks[0].exclude_paths == [first_file, nested_folder] + assert hooks[0].exclude_paths == [exclude_folder] + + +def test_restrict_folder_excludes_for_file_should_keep_file(fs: FakeFilesystem) -> None: + exclude_file = Path("Repo/excluded.py") + fs.create_file(exclude_file) + + restricted_hooks = restrict_folder_excludes([Hook("ruff", [exclude_file])], set()) + + assert restricted_hooks[0].exclude_paths == [exclude_file] + + +def test_restrict_folder_excludes_for_skipped_folder_should_keep_folder(fs: FakeFilesystem) -> None: + exclude_folder = Path("Repo/excluded") + fs.create_file(exclude_folder / "foo.py") + skipped_excludes = {SkippedExclude("ruff", exclude_folder)} + + restricted_hooks = restrict_folder_excludes([Hook("ruff", [exclude_folder])], skipped_excludes) + + assert restricted_hooks[0].exclude_paths == [exclude_folder] + + +def test_restrict_folder_excludes_should_only_skip_matching_hook_and_path(fs: FakeFilesystem) -> None: + shared_exclude_folder = Path("Repo/excluded") + excluded_file = shared_exclude_folder / "foo.py" + fs.create_file(excluded_file) + hooks = [Hook("ruff", [shared_exclude_folder]), Hook("black", [shared_exclude_folder])] + + restricted_hooks = restrict_folder_excludes(hooks, {SkippedExclude("ruff", shared_exclude_folder)}) + + assert restricted_hooks[0].exclude_paths == [shared_exclude_folder] + assert restricted_hooks[1].exclude_paths == [excluded_file] + + +def test_update_config_with_stricter_excludes_should_update_matching_hooks(fs: FakeFilesystem) -> None: + config_file = Path("Repo/.pre-commit-config.yaml") + fs.create_file( + config_file, + contents="""# Keep this comment. +repos: + - repo: local + hooks: + - id: ruff + exclude: | + (?x)^( + generated| + keep + ) + - id: black + exclude: 'unchanged.py' +""", + ) + + update_config_with_stricter_excludes( + config_file, + [Hook("ruff", [Path("Repo/generated/foo.py"), Path("Repo/generated/bar.py")])], + ) + + assert ( + config_file.read_text(encoding="utf-8") + == """# Keep this comment. +repos: + - repo: local + hooks: + - id: ruff + exclude: | + (?x)^( + generated/foo.py| + generated/bar.py + ) + - id: black + exclude: 'unchanged.py' +""" + )