From a3937012640e699a30471431e5bd8a58cef44472 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 21 Aug 2026 14:00:31 -0400 Subject: [PATCH] Speed up intersphinx resolutions --- CHANGES.rst | 5 ++ sphinx/ext/intersphinx/_resolve.py | 41 ++++++++++--- sphinx/ext/intersphinx/_shared.py | 53 ++++++++++++++++ .../test_ext_intersphinx.py | 60 ++++++++++++++++++- 4 files changed, 149 insertions(+), 10 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ad6d698341a..5da0d32a616 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,6 +12,11 @@ Bugs fixed does not match the language name, such as Chinese (which reuses the English stemmer) and Dutch (which uses the Dutch Porter stemmer). Patch by Hugo van Kemenade +* intersphinx: Use an index for the case-insensitive ``std:label`` and + ``std:term`` fallback lookup, instead of scanning every inventory entry + on each miss. This is speed-up for projects with many or large ``std`` + inventories. + Patch by Eric Larson. Release 9.1.0 (released Dec 31, 2025) diff --git a/sphinx/ext/intersphinx/_resolve.py b/sphinx/ext/intersphinx/_resolve.py index aa7a365df78..d20200391d0 100644 --- a/sphinx/ext/intersphinx/_resolve.py +++ b/sphinx/ext/intersphinx/_resolve.py @@ -28,7 +28,10 @@ from sphinx.domains import Domain from sphinx.domains._domains_container import _DomainsContainer from sphinx.environment import BuildEnvironment - from sphinx.ext.intersphinx._shared import InventoryName + from sphinx.ext.intersphinx._shared import ( + InventoryName, + _CaseInsensitiveIndex, + ) from sphinx.util.inventory import _InventoryItem from sphinx.util.typing import Inventory, RoleFunction @@ -80,6 +83,7 @@ def _create_element_from_result( def _resolve_reference_in_domain_by_target( inv_name: InventoryName | None, inventory: Inventory, + ci_index: _CaseInsensitiveIndex, domain_name: str, objtypes: Iterable[str], target: str, @@ -98,10 +102,7 @@ def _resolve_reference_in_domain_by_target( # Some types require case insensitive matches: # * 'term': https://github.com/sphinx-doc/sphinx/issues/9291 # * 'label': https://github.com/sphinx-doc/sphinx/issues/12008 - target_lower = target.lower() - insensitive_matches = list( - filter(lambda k: k.lower() == target_lower, inventory[objtype].keys()) - ) + insensitive_matches = ci_index.matches(objtype, target.lower()) if len(insensitive_matches) > 1: data_items = { inventory[objtype][match] for match in insensitive_matches @@ -144,6 +145,7 @@ def _resolve_reference_in_domain_by_target( def _resolve_reference_in_domain( inv_name: InventoryName | None, inventory: Inventory, + ci_index: _CaseInsensitiveIndex, honor_disabled_refs: bool, disabled_reftypes: Set[str], domain: Domain, @@ -177,7 +179,14 @@ def _resolve_reference_in_domain( # without qualification res = _resolve_reference_in_domain_by_target( - inv_name, inventory, domain_name, objtypes, node['reftarget'], node, contnode + inv_name, + inventory, + ci_index, + domain_name, + objtypes, + node['reftarget'], + node, + contnode, ) if res is not None: return res @@ -187,7 +196,14 @@ def _resolve_reference_in_domain( if full_qualified_name is None: return None return _resolve_reference_in_domain_by_target( - inv_name, inventory, domain_name, objtypes, full_qualified_name, node, contnode + inv_name, + inventory, + ci_index, + domain_name, + objtypes, + full_qualified_name, + node, + contnode, ) @@ -195,6 +211,7 @@ def _resolve_reference( inv_name: InventoryName | None, domains: _DomainsContainer, inventory: Inventory, + ci_index: _CaseInsensitiveIndex, honor_disabled_refs: bool, disabled_reftypes: Set[str], node: pending_xref, @@ -215,6 +232,7 @@ def _resolve_reference( res = _resolve_reference_in_domain( inv_name, inventory, + ci_index, honor_disabled_refs, disabled_reftypes, domain, @@ -244,6 +262,7 @@ def _resolve_reference( return _resolve_reference_in_domain( inv_name, inventory, + ci_index, honor_disabled_refs, disabled_reftypes, domain, @@ -270,10 +289,12 @@ def resolve_reference_in_inventory( Requires ``inventory_exists(env, inv_name)``. """ assert inventory_exists(env, inv_name) + inventories = InventoryAdapter(env) return _resolve_reference( inv_name, env.domains, - InventoryAdapter(env).named_inventory[inv_name], + inventories.named_inventory[inv_name], + inventories.case_insensitive_index(inv_name), False, frozenset(env.config.intersphinx_disabled_reftypes), node, @@ -291,10 +312,12 @@ def resolve_reference_any_inventory( Resolution is tried with the target as is in any inventory. """ + inventories = InventoryAdapter(env) return _resolve_reference( None, env.domains, - InventoryAdapter(env).main_inventory, + inventories.main_inventory, + inventories.case_insensitive_index(None), honor_disabled_refs, frozenset(env.config.intersphinx_disabled_reftypes), node, diff --git a/sphinx/ext/intersphinx/_shared.py b/sphinx/ext/intersphinx/_shared.py index cbedbf1b380..fb6575bcda3 100644 --- a/sphinx/ext/intersphinx/_shared.py +++ b/sphinx/ext/intersphinx/_shared.py @@ -2,6 +2,7 @@ from __future__ import annotations +import weakref from typing import TYPE_CHECKING from sphinx.util import logging @@ -111,6 +112,42 @@ def __delattr__(self, key: str) -> NoReturn: raise AttributeError(msg) +class _CaseInsensitiveIndex: + """Case-insensitive name index for a single inventory. + + Maps a lower-cased object name to the matching names in insertion order. + Only ``std:label`` and ``std:term`` are looked up case-insensitively, + so the per-object-type indices are built lazily, on first use. + """ + + __slots__ = ('_indices', '_inventory', '_sizes') + + def __init__(self, inventory: Inventory) -> None: + self._inventory = inventory + self._indices: dict[str, dict[str, list[str]]] = {} + self._sizes: dict[str, int] = {} + + def matches(self, objtype: str, target_lower: str) -> Sequence[str]: + """Names of *objtype* whose lower-cased form is *target_lower*.""" + objects = self._inventory[objtype] + index = self._indices.get(objtype) + if index is None or self._sizes[objtype] != len(objects): + # the size check catches inventories mutated after the index was built + index = {} + for name in objects: + index.setdefault(name.lower(), []).append(name) + self._indices[objtype] = index + self._sizes[objtype] = len(objects) + return index.get(target_lower, ()) + + +#: Case-insensitive indices, keyed by build environment. +#: Kept off the environment itself so that they are never pickled. +_CASE_INSENSITIVE_INDICES: weakref.WeakKeyDictionary[ + BuildEnvironment, dict[InventoryName | None, _CaseInsensitiveIndex] +] = weakref.WeakKeyDictionary() + + class InventoryAdapter: """Inventory adapter for environment""" @@ -143,6 +180,22 @@ def main_inventory(self) -> Inventory: def named_inventory(self) -> dict[InventoryName, Inventory]: return self.env.intersphinx_named_inventory # type: ignore[attr-defined] + def case_insensitive_index( + self, inv_name: InventoryName | None + ) -> _CaseInsensitiveIndex: + """Case-insensitive index for a named inventory, or the main inventory.""" + indices = _CASE_INSENSITIVE_INDICES.setdefault(self.env, {}) + try: + return indices[inv_name] + except KeyError: + if inv_name is None: + inventory = self.main_inventory + else: + inventory = self.named_inventory[inv_name] + index = indices[inv_name] = _CaseInsensitiveIndex(inventory) + return index + def clear(self) -> None: self.env.intersphinx_inventory.clear() # type: ignore[attr-defined] self.env.intersphinx_named_inventory.clear() # type: ignore[attr-defined] + _CASE_INSENSITIVE_INDICES.pop(self.env, None) diff --git a/tests/test_ext_intersphinx/test_ext_intersphinx.py b/tests/test_ext_intersphinx/test_ext_intersphinx.py index 5fcfe4d9260..9a7e9377f30 100644 --- a/tests/test_ext_intersphinx/test_ext_intersphinx.py +++ b/tests/test_ext_intersphinx/test_ext_intersphinx.py @@ -29,7 +29,10 @@ validate_intersphinx_mapping, ) from sphinx.ext.intersphinx._resolve import missing_reference -from sphinx.ext.intersphinx._shared import _IntersphinxProject +from sphinx.ext.intersphinx._shared import ( + _CaseInsensitiveIndex, + _IntersphinxProject, +) from sphinx.util.inventory import _Inventory, _InventoryItem from tests.test_util.intersphinx_data import ( @@ -370,6 +373,61 @@ def test_ambiguous_reference_handling(term, expected_ambiguity, tmp_path, app, w assert ambiguity is expected_ambiguity +@pytest.mark.sphinx('html', testroot='root') +def test_case_insensitive_match_uses_first_in_inventory_order(tmp_path, app): + inv_file = tmp_path / 'inventory' + inv_file.write_bytes(INVENTORY_V2_AMBIGUOUS_TERMS) + set_config(app, {'cmd': ('https://docs.python.org/', str(inv_file))}) + + validate_intersphinx_mapping(app, app.config) + load_mappings(app) + + # 'b term' precedes 'B term' in the inventory, so it wins + node, contnode = fake_node('std', 'term', 'B TERM', 'B TERM') + rn = missing_reference(app, app.env, node, contnode) + assert rn['refuri'] == 'https://docs.python.org/document.html#id5' + + +@pytest.mark.sphinx('html', testroot='root') +def test_case_insensitive_match_after_reload(tmp_path, app): + inv_file = tmp_path / 'inventory' + inv_file.write_bytes(INVENTORY_V2) + set_config(app, {'cmd': ('https://docs.python.org/', str(inv_file))}) + + validate_intersphinx_mapping(app, app.config) + load_mappings(app) + + node, contnode = fake_node('std', 'ref', 'the-julia-domain', 'the-julia-domain') + assert missing_reference(app, app.env, node, contnode) is not None + + # reloading with a different inventory must not leave a stale index behind + other_inv_file = tmp_path / 'other-inventory' + other_inv_file.write_bytes(INVENTORY_V2_AMBIGUOUS_TERMS) + set_config(app, {'cmd': ('https://example.org/', str(other_inv_file))}) + validate_intersphinx_mapping(app, app.config) + load_mappings(app) + + node, contnode = fake_node('std', 'ref', 'the-julia-domain', 'the-julia-domain') + assert missing_reference(app, app.env, node, contnode) is None + + node, contnode = fake_node('std', 'term', 'A TERM', 'A TERM') + rn = missing_reference(app, app.env, node, contnode) + assert rn['refuri'] == 'https://example.org/glossary.html#term-a-term' + + +def test_case_insensitive_index_tracks_inventory_changes(): + item = _InventoryItem( + project_name='foo', project_version='1', uri='u', display_name='-' + ) + inventory: Inventory = {'std:label': {'Spam': item}} + index = _CaseInsensitiveIndex(inventory) + assert index.matches('std:label', 'spam') == ['Spam'] + assert index.matches('std:label', 'eggs') == () + + inventory['std:label']['EGGS'] = item + assert index.matches('std:label', 'eggs') == ['EGGS'] + + @pytest.mark.sphinx('html', testroot='ext-intersphinx-cppdomain') def test_missing_reference_cppdomain(tmp_path, app): inv_file = tmp_path / 'inventory'