From 8d81ca2d770c187dee725a33f655b2ca6bf466de Mon Sep 17 00:00:00 2001 From: Mauricio Villegas <5780272+mauvilsa@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:01:10 +0200 Subject: [PATCH 1/2] Add #key.path selector to config file paths, e.g. pyproject.toml#tool.myapp --- CHANGELOG.rst | 4 + DOCUMENTATION.rst | 9 + jsonargparse/_actions.py | 10 +- jsonargparse/_core.py | 69 ++++--- jsonargparse/_formatters.py | 4 +- jsonargparse/_namespace.py | 1 + jsonargparse/_util.py | 37 ++++ jsonargparse_tests/test_config_selector.py | 198 +++++++++++++++++++++ 8 files changed, 306 insertions(+), 26 deletions(-) create mode 100644 jsonargparse_tests/test_config_selector.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 72a722e0..0c23e453 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -68,6 +68,10 @@ Added - New ``extension_or_*`` parser modes, e.g. ``extension_or_yaml``, which load each config file in the format given by its extension, and otherwise as the given mode (`#1021 `__). +- Config file paths accept a ``#key.path`` suffix to parse only that part of the + file, e.g. ``pyproject.toml#tool.myapp``, in default config files, the config + argument and ``parse_path``. Default config files without the key are skipped + (`#1022 `__). Fixed ^^^^^ diff --git a/DOCUMENTATION.rst b/DOCUMENTATION.rst index 24280336..4d03b327 100644 --- a/DOCUMENTATION.rst +++ b/DOCUMENTATION.rst @@ -1378,6 +1378,15 @@ All matching files are parsed in the given order and override the defaults from the source code. They are always parsed first, so any command line argument overrides their values. +Only part of a config file can be parsed by appending ``#`` and a dot separated +key, e.g. ``pyproject.toml#tool.myapp`` or ``package.json#myapp``, for files +shared with other tools. This works for default config files, the config +argument and :py:meth:`.ArgumentParser.parse_path`. A default config file that +does not have the key is skipped, otherwise it is an error. Combined with an +``extension_or_`` parser mode, an app can read ``pyproject.toml`` and also its +own YAML config, e.g. +``default_config_files=['pyproject.toml#tool.myapp', '.myapp.yaml']``. + An argument can also be added to give a config file path explicitly. This does not disable ``default_config_files``. The config argument is parsed at its position among the command line arguments, so arguments after it override the diff --git a/jsonargparse/_actions.py b/jsonargparse/_actions.py index 110910bb..e4efc1b3 100644 --- a/jsonargparse/_actions.py +++ b/jsonargparse/_actions.py @@ -20,7 +20,7 @@ ) from ._loaders_dumpers import get_loader_exceptions, load_value from ._namespace import Namespace, copy_provenance -from ._optionals import _get_config_read_mode, ruamel_support +from ._optionals import ruamel_support from ._paths import cwd_relative_path from ._type_checking import ArgumentParser from ._util import ( @@ -29,6 +29,7 @@ argument_error, config_file_context, default_config_option_help, + get_config_path, get_import_path, import_object, indent_text, @@ -127,8 +128,10 @@ def apply_config(parser, cfg, dest, value) -> None: with parser_context(single_subcommand=False), previous_config_context(cfg), skip_apply_links(): kwargs = {"env": False, "defaults": False, "_skip_validation": True, "_fail_no_subcommand": False} + cfg_path: Path | None + selector: str | None = None try: - cfg_path: Path | None = Path(value, mode=_get_config_read_mode()) + cfg_path, selector = get_config_path(value) except TypeError as ex_path: try: try: @@ -151,7 +154,8 @@ def apply_config(parser, cfg, dest, value) -> None: copy_provenance(cfg_merged, cfg) if cfg.get(dest) is get_parsing_setting("unset_sentinel"): cfg[dest] = [] - cfg[dest].append(None if cfg_path is None else cwd_relative_path(cfg_path)) + config_path = None if cfg_path is None else cwd_relative_path(cfg_path) + cfg[dest].append(f"{config_path}#{selector}" if selector else config_path) def completer(self, prefix, **kwargs): from ._completions import get_files_completer diff --git a/jsonargparse/_core.py b/jsonargparse/_core.py index 5b6ef1f5..753deb14 100644 --- a/jsonargparse/_core.py +++ b/jsonargparse/_core.py @@ -108,11 +108,13 @@ ) from ._util import ( ComposedConfig, + ConfigKeyNotFoundError, Path, argument_error, check_no_composed_config, config_file_context, get_argument_group_class, + get_config_path, get_private_kwargs, identity, indent_text, @@ -120,6 +122,7 @@ merge_config, resolve_config_includes, return_parser_if_captured, + select_config_key, ) __all__ = ["ArgumentParser"] @@ -338,7 +341,8 @@ def __init__( ``jsonnet``, ``omegaconf`` or one added via :func:`.set_loader`. With an ``extension_or_`` prefix, config files are loaded in the format given by their extension. dump_header: Header to include as comment when dumping a config object. - default_config_files: Default config file locations, e.g. ``['~/.config/myapp/*.yaml']``. + default_config_files: Default config file locations, e.g. ``['~/.config/myapp/*.yaml']``, optionally + with a ``#key.path`` to parse only that part, e.g. ``pyproject.toml#tool.myapp``. default_env: Whether to parse environment variables by default. allow_abbrev: Whether to accept abbreviated long options, ``None`` to use the global setting. """ @@ -737,7 +741,8 @@ def parse_path( """Parses a configuration file given its path. Args: - path: Path to the configuration file to parse. + path: Path to the configuration file to parse, optionally with a ``#key.path`` to parse only that + part, e.g. ``pyproject.toml#tool.myapp``. ext_vars: External variables for parsing Jsonnet. env: Whether to merge with the parsed environment, ``None`` to use the parser's default. defaults: Whether to merge with the parser's defaults. @@ -749,16 +754,16 @@ def parse_path( ArgumentError: If the parsing fails and ``exit_on_error=False``. TypeError: If the path does not exist or is not readable. """ - fpath = Path(path, mode=_get_config_read_mode()) + fpath, selector = get_config_path(path) with load_config_path_context(fpath), path_dir_context(fpath): content = fpath.read_text() parsed_cfg = self.parse_string( content=content, - path=os.path.basename(urlsplit(fpath.absolute).path if fpath.is_url else path), + path=os.path.basename(urlsplit(fpath.absolute).path if fpath.is_url else fpath.relative), ext_vars=ext_vars, env=env, defaults=defaults, - _source=ValueSource("config file", fpath, self.parser_mode), + _source=ValueSource("config file", fpath, self.parser_mode, selector=selector), **kwargs, ) @@ -795,7 +800,9 @@ def parse_string( try: with parser_context(load_value_mode=self.parser_mode), value_source_context(source): - cfg = self._load_config_parser_mode(content, path, ext_vars, previous_config.get()) + cfg = self._load_config_parser_mode( + content, path, ext_vars, previous_config.get(), selector=source.selector + ) if defaults or env: cfg_base = self._parse_defaults_and_environ(defaults, env) @@ -821,6 +828,7 @@ def _load_config_parser_mode( path: str | os.PathLike = "", ext_vars: dict | None = None, prev_cfg: Namespace | None = None, + selector: str | None = None, ) -> Namespace: """Loads a configuration string into a namespace. @@ -828,9 +836,11 @@ def _load_config_parser_mode( content: The configuration content. path: Optional path to original config path, just for error printing. ext_vars: Optional external variables used for parsing jsonnet. + selector: Optional ``key.path`` of the part of the config to load. Raises: TypeError: If there is an invalid value according to the parser. + ConfigKeyNotFoundError: If the selected key is not in the config. """ try: cfg_dict = load_config_value(content, path, path=path, ext_vars=ext_vars) @@ -838,6 +848,8 @@ def _load_config_parser_mode( raise TypeError(f"Problems parsing config:\n{indent_text(str(ex))}") from ex if not isinstance(cfg_dict, dict): raise TypeError(f"Unexpected config: {content}") + if selector is not None: + cfg_dict = select_config_key(cfg_dict, selector) return self._apply_actions(resolve_config_includes(cfg_dict), prev_cfg=prev_cfg) ## Methods for adding to the parser ## @@ -1124,18 +1136,24 @@ def save_paths(cfg): ## Methods related to defaults ## - def _get_default_config_files(self) -> list[Path]: + def _get_default_config_files(self) -> list[tuple[Path, str | None]]: + """Returns the existing default config files, each with the key selected in it with ``path#key``.""" if getattr(self, "_inner_parser", False): return [] - default_config_files = [] + default_config_files: list[tuple[str, str | None]] = [] for pattern in self.default_config_files: - default_config_files += sorted(glob.glob(os.path.expanduser(pattern))) + matches = sorted(glob.glob(os.path.expanduser(pattern))) + file_pattern, _, selector = pattern.rpartition("#") + if matches or not (file_pattern and selector): + default_config_files += [(m, None) for m in matches] + else: + default_config_files += [(m, selector) for m in sorted(glob.glob(os.path.expanduser(file_pattern)))] if len(default_config_files) > 0: with suppress(TypeError): - return [Path(v, mode=_get_config_read_mode()) for v in default_config_files] + return [(Path(v, mode=_get_config_read_mode()), s) for v, s in default_config_files] return [] def get_default(self, dest: str) -> Any: @@ -1187,7 +1205,8 @@ def get_defaults(self, skip_validation: bool = False) -> Namespace: self._logger.debug("Loaded parser defaults: %s", cfg) default_config_files = self._get_default_config_files() - for default_config_file in default_config_files: + for default_config_file, selector in default_config_files: + name = cwd_relative_path(default_config_file) + (f"#{selector}" if selector else "") with ( load_config_path_context(default_config_file), path_dir_context(default_config_file), @@ -1196,11 +1215,18 @@ def get_defaults(self, skip_validation: bool = False) -> Namespace: default_config_file_content = default_config_file.read_text() if not default_config_file_content.strip(): continue - source = ValueSource("default config file", default_config_file, self.parser_mode) + source = ValueSource("default config file", default_config_file, self.parser_mode, selector=selector) with value_source_context(source): - cfg_file = self._load_config_parser_mode( - default_config_file_content, os.path.basename(default_config_file), prev_cfg=cfg - ) + try: + cfg_file = self._load_config_parser_mode( + default_config_file_content, + os.path.basename(default_config_file), + prev_cfg=cfg, + selector=selector, + ) + except ConfigKeyNotFoundError as ex: + self._logger.debug("Skipped default config file %s: %s", name, ex) + continue cfg = merge_config(self, cfg_file, cfg) try: with _ActionPrintConfig.skip_print_config(): @@ -1214,18 +1240,17 @@ def get_defaults(self, skip_validation: bool = False) -> Namespace: ) except (TypeError, KeyError, argparse.ArgumentError) as ex: raise argument_error( - f"Problem in default config file '{default_config_file}': {ex.args[0]}", - default_config_file=str(default_config_file), + f"Problem in default config file '{name}': {ex.args[0]}", + default_config_file=str(name), ) from ex meta = cfg.get("__default_config__") - config_path = cwd_relative_path(default_config_file) if isinstance(meta, list): - meta.append(config_path) + meta.append(name) elif isinstance(meta, str): - cfg["__default_config__"] = [meta, config_path] + cfg["__default_config__"] = [meta, name] else: - cfg["__default_config__"] = config_path - self._logger.debug("Parsed default configuration from path: %s", default_config_file) + cfg["__default_config__"] = name + self._logger.debug("Parsed default configuration from path: %s", name) with parser_context(validating_defaults=True): ActionTypeHint.add_sub_defaults(self, cfg) diff --git a/jsonargparse/_formatters.py b/jsonargparse/_formatters.py index cd472f59..6cd0dcdf 100644 --- a/jsonargparse/_formatters.py +++ b/jsonargparse/_formatters.py @@ -346,6 +346,8 @@ def describe_source(source, cache: dict | None = None) -> str: if source.implied_by is not None: return f"{source.description}, from {describe_source(source.implied_by, cache)}" text = source.description if source.origin is None else f"{source.description} {_describe_origin(source.origin)}" + if source.selector: + text += f"#{source.selector}" position = _find_position(source, {} if cache is None else cache) return text if position is None else f"{text}:{position[0]}" @@ -380,7 +382,7 @@ def _find_position(source, cache: dict) -> tuple[int, int, str] | None: content = source.origin.read_text() cache[id(source.origin)] = (_get_config_key_positions(content), content.splitlines()) positions, lines = cache[id(source.origin)] - key = source.key + key = f"{source.selector}.{source.key}" if source.selector else source.key while key: for candidate in [key, ("." + key).replace(".init_args.", ".")[1:]]: # init_args can be implicit if candidate in positions: diff --git a/jsonargparse/_namespace.py b/jsonargparse/_namespace.py index f54a0bed..f42fa832 100644 --- a/jsonargparse/_namespace.py +++ b/jsonargparse/_namespace.py @@ -23,6 +23,7 @@ class ValueSource(NamedTuple): mode: str | None = None # the parser mode of a config file, required to find line numbers key: str | None = None # the key in a config file that the value corresponds to implied_by: "ValueSource | None" = None # for implicit values, e.g. a class_path, the source of what implied it + selector: str | None = None # the key selected in a config file with path#key, of which key is relative def __deepcopy__(self, memo): return self # immutable, so copies of namespaces can share it, which is much faster, e.g. for list items diff --git a/jsonargparse/_util.py b/jsonargparse/_util.py index 2acd094a..36fcf68b 100644 --- a/jsonargparse/_util.py +++ b/jsonargparse/_util.py @@ -76,6 +76,43 @@ def merge_config(parser, source: Namespace, target: Namespace) -> Namespace: return target +class ConfigKeyNotFoundError(TypeError): + """Raised when the key selected in a config with ``path#key`` is not in it.""" + + +def get_config_path(value: str | os.PathLike) -> tuple[Path, str | None]: + """Returns the path of a config file and the key selected in it with ``path#key.path``, if any. + + The value is split on the last ``#`` only if it is not a path itself but the part before is. + + Raises: + TypeError: If the value is not a path to a config file. + """ + mode = _get_config_read_mode() + try: + return Path(value, mode=mode), None + except TypeError as ex: + path, _, selector = os.fspath(value).rpartition("#") + if not path or not selector: + raise + try: + return Path(path, mode=mode), selector + except TypeError: + raise ex from None + + +def select_config_key(cfg: dict, selector: str) -> dict: + """Returns the mapping at a ``key.path`` selected in a loaded config.""" + value: Any = cfg + for key in selector.split("."): + if not isinstance(value, dict) or key not in value: + raise ConfigKeyNotFoundError(f'Key "{selector}" not found in the config') + value = value[key] + if not isinstance(value, dict): + raise TypeError(f'Expected a mapping at key "{selector}". Got value: {value}') + return value + + def _config_path_id(cfg_path: Path) -> tuple[str, str]: path_id = cfg_path.absolute if not (cfg_path.is_url or cfg_path.is_fsspec): diff --git a/jsonargparse_tests/test_config_selector.py b/jsonargparse_tests/test_config_selector.py new file mode 100644 index 00000000..784194af --- /dev/null +++ b/jsonargparse_tests/test_config_selector.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import json +import os +import pathlib + +import pytest + +from jsonargparse import ArgumentError, ArgumentParser +from jsonargparse._formatters import describe_source +from jsonargparse._namespace import get_provenance +from jsonargparse._optionals import toml_load_available +from jsonargparse_tests.conftest import capture_logs, get_parser_help, skip_if_no_pyyaml + + +@pytest.fixture +def config_parser() -> ArgumentParser: + parser = ArgumentParser(exit_on_error=False, parser_mode="json") + parser.add_argument("--config", action="config") + parser.add_argument("--a", type=int, default=0) + parser.add_argument("--b", type=int, default=0) + return parser + + +def write(path: str, data) -> None: + pathlib.Path(path).write_text(data if isinstance(data, str) else json.dumps(data)) + + +def get_sources(cfg) -> dict: + return { + key: describe_source(source) + for key, source in get_provenance(cfg).items() + if key.rsplit(".", 1)[-1] != "config" + } + + +def get_error(parser: ArgumentParser, args: list[str]) -> str: + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(args) + return str(ctx.value) + + +# config argument and parse_path + + +def test_selector_config_argument(config_parser, tmp_cwd): + write("package.json", {"name": "pkg", "myapp": {"a": 1}}) + cfg = config_parser.parse_args(["--config=package.json#myapp"]) + assert (cfg.a, cfg.b) == (1, 0) + assert cfg.config == ["package.json#myapp"] + assert config_parser.get_config_files(cfg) == ["package.json#myapp"] + + +def test_selector_config_path_relative_to_cwd(config_parser, tmp_path, monkeypatch): + (tmp_path / "configs").mkdir() + (tmp_path / "work").mkdir() + (tmp_path / "configs" / "main.json").write_text('{"config": "package.json#myapp"}') + (tmp_path / "configs" / "package.json").write_text('{"myapp": {"a": 1}}') + monkeypatch.chdir(tmp_path / "work") + cfg = config_parser.parse_args(["--config=../configs/main.json"]) + assert cfg.a == 1 + assert cfg.config == [ + os.path.join("..", "configs", "package.json") + "#myapp", + os.path.join("..", "configs", "main.json"), + ] + + +def test_selector_nested_key(config_parser, tmp_cwd): + write("Cargo.json", {"package": {"metadata": {"myapp": {"b": 2}}, "version": "1"}}) + cfg = config_parser.parse_args(["--config=Cargo.json#package.metadata.myapp"]) + assert cfg.b == 2 + + +def test_selector_parse_path(config_parser, tmp_cwd): + write("package.json", {"myapp": {"a": 3}}) + assert config_parser.parse_path("package.json#myapp").a == 3 + + +def test_selector_missing_key_error(config_parser, tmp_cwd): + write("package.json", {"name": "pkg"}) + error = get_error(config_parser, ["--config=package.json#myapp"]) + assert 'Key "myapp" not found in the config' in error + assert error.endswith("Source: config file package.json#myapp") + + +def test_selector_not_a_mapping_error(config_parser, tmp_cwd): + write("package.json", {"myapp": [1, 2]}) + error = get_error(config_parser, ["--config=package.json#myapp"]) + assert 'Expected a mapping at key "myapp". Got value: [1, 2]' in error + + +def test_selector_file_name_with_hash(config_parser, tmp_cwd): + write("a#b.json", {"a": 1}) + write("c#d.json", {"myapp": {"a": 2}}) + assert config_parser.parse_args(["--config=a#b.json"]).a == 1 + assert config_parser.parse_args(["--config=c#d.json#myapp"]).a == 2 + + +def test_selector_file_not_found(config_parser, tmp_cwd): + error = get_error(config_parser, ["--config=missing.json#myapp"]) + assert "Not a valid path nor a config string" in error + assert "missing.json#myapp" in error + + +def test_selector_empty_key(config_parser, tmp_cwd): + write("package.json", {"a": 1}) + error = get_error(config_parser, ["--config=package.json#"]) + assert "Not a valid path nor a config string" in error + + +# default_config_files + + +def test_selector_default_config_files_precedence(config_parser, tmp_cwd): + write("user.json", {"a": 1, "b": 1}) + write("package.json", {"myapp": {"a": 2, "b": 2}}) + write("app.json", {"b": 3}) + config_parser.default_config_files = ["user.json", "package.json#myapp", "app.json"] + cfg = config_parser.parse_args([]) + assert (cfg.a, cfg.b) == (2, 3) + assert cfg.__default_config__ == ["user.json", "package.json#myapp", "app.json"] + assert get_sources(cfg) == { + "a": "default config file package.json#myapp", + "b": "default config file app.json", + } + + +def test_selector_default_config_file_only(config_parser, tmp_cwd): + write("package.json", {"myapp": {"a": 2}}) + config_parser.default_config_files = ["package.json#myapp"] + cfg = config_parser.parse_args([]) + assert cfg.a == 2 + assert cfg.__default_config__ == "package.json#myapp" + help_str = get_parser_help(config_parser) + assert "overridden by the contents of: package.json#myapp" in help_str + + +def test_selector_default_config_file_missing_key_skipped(config_parser, tmp_cwd, logger): + write("package.json", {"name": "pkg"}) + config_parser.logger = logger + config_parser.default_config_files = ["package.json#myapp"] + with capture_logs(logger) as logs: + cfg = config_parser.parse_args([]) + assert cfg.a == 0 + assert "__default_config__" not in cfg + assert 'Skipped default config file package.json#myapp: Key "myapp" not found' in logs.getvalue() + + +def test_selector_default_config_files_pattern(config_parser, tmp_cwd): + write("one.json", {"myapp": {"a": 1}}) + write("two.json", {"myapp": {"b": 2}}) + config_parser.default_config_files = ["*.json#myapp"] + cfg = config_parser.parse_args([]) + assert (cfg.a, cfg.b) == (1, 2) + assert cfg.__default_config__ == ["one.json#myapp", "two.json#myapp"] + + +def test_selector_default_config_file_error(config_parser, tmp_cwd): + write("package.json", {"myapp": {"a": "abc"}}) + config_parser.default_config_files = ["package.json#myapp"] + error = get_error(config_parser, []) + assert "Expected type: int. Got value: abc" in error + assert error.endswith("Source: default config file package.json#myapp") + + +def test_selector_default_config_file_problem(config_parser, tmp_cwd): + write("package.json", {"myapp": {"unknown": 1}}) + config_parser.default_config_files = ["package.json#myapp"] + with pytest.raises(ArgumentError) as ctx: + config_parser.get_defaults() + assert str(ctx.value).startswith("Problem in default config file 'package.json#myapp'") + + +# formats + + +@pytest.mark.skipif(not toml_load_available, reason="tomllib or tomli package is required") +def test_selector_pyproject_toml(tmp_cwd): + parser = ArgumentParser(exit_on_error=False, parser_mode="extension_or_json") + parser.add_argument("--config", action="config") + parser.add_argument("--line_length", type=int, default=80) + write("pyproject.toml", '[project]\nname = "pkg"\n\n[tool.myapp]\nline_length = 100\n') + parser.default_config_files = ["pyproject.toml#tool.myapp"] + assert parser.parse_args([]).line_length == 100 + + +@skip_if_no_pyyaml +def test_selector_yaml_line_numbers(tmp_cwd): + parser = ArgumentParser(exit_on_error=False, parser_mode="yaml") + parser.add_argument("--config", action="config") + parser.add_argument("--a", type=int, default=0) + parser.add_argument("--b", type=int, default=0) + write("services.yaml", "web:\n a: 1\napi:\n a: 2\n b: 3\n") + cfg = parser.parse_args(["--config=services.yaml#api"]) + assert get_sources(cfg) == { + "a": "config file services.yaml#api:4", + "b": "config file services.yaml#api:5", + } From dae97c2f277ed07c11eaad0a7009bb6fe7d08f38 Mon Sep 17 00:00:00 2001 From: Mauricio Villegas <5780272+mauvilsa@users.noreply.github.com> Date: Wed, 21 Oct 2026 07:13:33 +0200 Subject: [PATCH 2/2] Fix issues --- DOCUMENTATION.rst | 7 +++-- jsonargparse/_core.py | 4 +-- jsonargparse/_util.py | 25 +++++++++------- jsonargparse_tests/test_config_selector.py | 34 +++++++++++++++++++--- 4 files changed, 51 insertions(+), 19 deletions(-) diff --git a/DOCUMENTATION.rst b/DOCUMENTATION.rst index 4d03b327..0529ff74 100644 --- a/DOCUMENTATION.rst +++ b/DOCUMENTATION.rst @@ -1382,9 +1382,10 @@ Only part of a config file can be parsed by appending ``#`` and a dot separated key, e.g. ``pyproject.toml#tool.myapp`` or ``package.json#myapp``, for files shared with other tools. This works for default config files, the config argument and :py:meth:`.ArgumentParser.parse_path`. A default config file that -does not have the key is skipped, otherwise it is an error. Combined with an -``extension_or_`` parser mode, an app can read ``pyproject.toml`` and also its -own YAML config, e.g. +does not have the key is skipped, otherwise it is an error. A selected part +cannot load another part of the same file. Combined with an ``extension_or_`` +parser mode, an app can read ``pyproject.toml`` and also its own YAML config, +e.g. ``default_config_files=['pyproject.toml#tool.myapp', '.myapp.yaml']``. An argument can also be added to give a config file path explicitly. This does diff --git a/jsonargparse/_core.py b/jsonargparse/_core.py index 753deb14..32d33757 100644 --- a/jsonargparse/_core.py +++ b/jsonargparse/_core.py @@ -755,7 +755,7 @@ def parse_path( TypeError: If the path does not exist or is not readable. """ fpath, selector = get_config_path(path) - with load_config_path_context(fpath), path_dir_context(fpath): + with load_config_path_context(fpath, selector), path_dir_context(fpath): content = fpath.read_text() parsed_cfg = self.parse_string( content=content, @@ -1208,7 +1208,7 @@ def get_defaults(self, skip_validation: bool = False) -> Namespace: for default_config_file, selector in default_config_files: name = cwd_relative_path(default_config_file) + (f"#{selector}" if selector else "") with ( - load_config_path_context(default_config_file), + load_config_path_context(default_config_file, selector), path_dir_context(default_config_file), parser_context(parent_parser=self, parsing_defaults=True), ): diff --git a/jsonargparse/_util.py b/jsonargparse/_util.py index 36fcf68b..2f4a3b9f 100644 --- a/jsonargparse/_util.py +++ b/jsonargparse/_util.py @@ -44,7 +44,7 @@ default_config_option_help = "Path to a configuration file." -config_load_stack: ContextVar[tuple[tuple[str, str], ...]] = ContextVar("config_load_stack", default=()) +config_load_stack: ContextVar[tuple[tuple[str, str, str | None], ...]] = ContextVar("config_load_stack", default=()) def argument_error(message: str, default_config_file: str | None = None) -> ArgumentError: @@ -113,32 +113,37 @@ def select_config_key(cfg: dict, selector: str) -> dict: return value -def _config_path_id(cfg_path: Path) -> tuple[str, str]: +def _config_path_id(cfg_path: Path, selector: str | None) -> tuple[str, str, str | None]: path_id = cfg_path.absolute if not (cfg_path.is_url or cfg_path.is_fsspec): path_id = os.path.realpath(path_id) - return path_id, str(cfg_path) + return path_id, str(cfg_path) + (f"#{selector}" if selector else ""), selector -def _format_config_load_chain(stack: tuple[tuple[str, str], ...], path_id: tuple[str, str]) -> str: +def _format_config_load_chain( + stack: tuple[tuple[str, str, str | None], ...], path_id: tuple[str, str, str | None] +) -> str: chain = list(stack) + [path_id] - for num, (stack_path, _) in enumerate(chain): + for num, (stack_path, _, _) in enumerate(chain): if stack_path == path_id[0]: chain = chain[num:] break - return " -> ".join(display for _, display in chain) + return " -> ".join(display for _, display, _ in chain) @contextmanager -def load_config_path_context(cfg_path: Path | None) -> Iterator[None]: +def load_config_path_context(cfg_path: Path | None, selector: str | None = None) -> Iterator[None]: if cfg_path is None: yield return - path_id = _config_path_id(cfg_path) + path_id = _config_path_id(cfg_path, selector) stack = config_load_stack.get() - if path_id[0] in {path for path, _ in stack}: + selectors = {sel for path, _, sel in stack if path == path_id[0]} + if selectors: chain = _format_config_load_chain(stack, path_id) - raise TypeError(f"Config file loop detected: {chain}") + if selector in selectors: + raise TypeError(f"Config file loop detected: {chain}") + raise TypeError(f"Loading another section of the same config file is not supported: {chain}") token = config_load_stack.set(stack + (path_id,)) try: yield diff --git a/jsonargparse_tests/test_config_selector.py b/jsonargparse_tests/test_config_selector.py index 784194af..4cad6907 100644 --- a/jsonargparse_tests/test_config_selector.py +++ b/jsonargparse_tests/test_config_selector.py @@ -59,10 +59,7 @@ def test_selector_config_path_relative_to_cwd(config_parser, tmp_path, monkeypat monkeypatch.chdir(tmp_path / "work") cfg = config_parser.parse_args(["--config=../configs/main.json"]) assert cfg.a == 1 - assert cfg.config == [ - os.path.join("..", "configs", "package.json") + "#myapp", - os.path.join("..", "configs", "main.json"), - ] + assert cfg.config == [os.path.join("..", "configs", "package.json") + "#myapp", "../configs/main.json"] def test_selector_nested_key(config_parser, tmp_cwd): @@ -102,6 +99,35 @@ def test_selector_file_not_found(config_parser, tmp_cwd): assert "missing.json#myapp" in error +def test_selector_section_of_other_file(config_parser, tmp_cwd): + write("shared.json", {"base": {"a": 1, "b": 2}}) + write("main.json", {"app": {"config": "shared.json#base", "b": 3}}) + cfg = config_parser.parse_args(["--config=main.json#app"]) + assert (cfg.a, cfg.b) == (1, 3) + assert cfg.config == ["shared.json#base", "main.json#app"] + + +def test_selector_same_file_other_section_error(config_parser, tmp_cwd): + write("shared.json", {"base": {"a": 1}, "app": {"config": "shared.json#base"}}) + error = get_error(config_parser, ["--config=shared.json#app"]) + assert "Loading another section of the same config file is not supported" in error + assert "shared.json#app -> shared.json#base" in error + + +def test_selector_whole_file_to_own_section_error(config_parser, tmp_cwd): + write("shared.json", {"config": "shared.json#base", "base": {"a": 1}}) + error = get_error(config_parser, ["--config=shared.json"]) + assert "Loading another section of the same config file is not supported" in error + assert "shared.json -> shared.json#base" in error + + +def test_selector_same_section_loop_error(config_parser, tmp_cwd): + write("one.json", {"app": {"config": "two.json"}}) + write("two.json", {"config": "one.json#app"}) + error = get_error(config_parser, ["--config=one.json#app"]) + assert "Config file loop detected: one.json#app -> two.json -> one.json#app" in error + + def test_selector_empty_key(config_parser, tmp_cwd): write("package.json", {"a": 1}) error = get_error(config_parser, ["--config=package.json#"])