From 738a24d314207f504301aae80474a900def53aaf Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 19:51:51 +0000 Subject: [PATCH] Document that only the first argument of memoize_path is fingerprinted Changes to other files the function reads (e.g., BIDS sidecars) do not invalidate cached results, in stat() mode and with a custom fingerprint alike; pair_with_path keeps results separate per location but does not make them depend on neighbors. Closes #114. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014CXyw6SCZnT8aLws6zZ8RE --- docs/design/git-annex-content.md | 25 +++++++++++++++++++++++++ src/fscacher/annex.py | 4 +++- src/fscacher/cache.py | 6 ++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/design/git-annex-content.md b/docs/design/git-annex-content.md index dbb2252..622185a 100644 --- a/docs/design/git-annex-content.md +++ b/docs/design/git-annex-content.md @@ -86,6 +86,9 @@ dereferenced (dereferencing would give the object path, identical for all files with the key). Results are then only shared by files at the same path, which is needed whenever the result depends on the path, e.g. on the extension (the key's extension may differ from the file's) or on neighboring files. +Pairing is not *sufficient* for the latter, though: it keeps results separate +per location, but does not make them change when a neighbor does (see +[Other files the function reads](#other-files-the-function-reads)). The path is made absolute with `os.path.abspath`, which does not resolve symlinks (neither the file's nor its parent directories'), so the same file reached through different paths gets separate entries; that errs on the safe @@ -95,6 +98,28 @@ With `pair_with_path=False`, the fingerprint is the key alone, and results are shared by all files with the same content: twins in a dataset, and the same file across clones. Only use it for results that depend on the content only. +## Other files the function reads + +**Only the first argument is fingerprinted**, in `stat()` mode and with a +custom fingerprint alike. If the decorated function also reads other files, +e.g. the BIDS sidecar `sub-01_bold.json` of `sub-01_bold.nii.gz`, or metadata +inherited from parent directories, changes to those files do not change the +cache key, so the stale result is returned. This predates custom +fingerprints and is not specific to git-annex; `pair_with_path` does not +change it, as the pair `(path, key of the .nii.gz)` stays the same when only +the sidecar changes. + +To have such changes noticed, either: + +- fold fingerprints of the relevant files into a custom fingerprint, e.g. + `(annex_key_fingerprint(path), *(fingerprints of the sidecars))`, keeping in + mind that the callable runs on every call and must stay cheap, and that it + must change whenever the result may (no "modified just now" window applies); + or +- have the caller pass something that identifies the content of those files + (e.g. their keys or `stat()`s, or the parsed sidecar itself) as extra + arguments of the decorated function, which are part of the cache key. + ## Dropped content With `stat()`, a locked file whose content was dropped is a broken symlink: it diff --git a/src/fscacher/annex.py b/src/fscacher/annex.py index f7591c8..dd9b0bf 100644 --- a/src/fscacher/annex.py +++ b/src/fscacher/annex.py @@ -36,7 +36,9 @@ def annex_key_fingerprint(path, *, pair_with_path=True): (as given, not dereferenced) and the key, so results are only shared by files at the same path, as needed when they depend on the path (e.g., on the extension or on neighboring files). If false, results are shared by - all files with the same key, e.g., across clones of a dataset. + all files with the same key, e.g., across clones of a dataset. Either + way, only this file is fingerprinted: changes to neighboring files (e.g., + sidecars) do not invalidate cached results. """ try: path = os.fsdecode(os.fspath(path)) diff --git a/src/fscacher/cache.py b/src/fscacher/cache.py index 3739565..1ea0115 100644 --- a/src/fscacher/cache.py +++ b/src/fscacher/cache.py @@ -101,6 +101,12 @@ def memoize_path(self, f=None, *, exclude_kwargs=None, custom_fingerprint=None): `None` to fall back to ``stat()``. For a directory, it may return a fingerprint of the whole tree, or `None` to fingerprint it by ``stat()``-ing each file as usual. + + Only the first argument is fingerprinted, with or without + ``custom_fingerprint``: changes to other files the function reads + (e.g., sidecar files next to it) do not invalidate cached results. + Pass something identifying their content as extra arguments, or fold + their fingerprints into a ``custom_fingerprint``, to have them noticed. """ if f is None: return partial(