diff --git a/.github/workflows/test-impl.yml b/.github/workflows/test-impl.yml index 6ad1ec956e9..254f6e92684 100644 --- a/.github/workflows/test-impl.yml +++ b/.github/workflows/test-impl.yml @@ -161,6 +161,8 @@ jobs: - run: mise run test:e2e e2e/cli/test_dotfiles_rollback - run: mise run test:e2e e2e/cli/test_dotfiles_rollback_types - run: mise run test:e2e e2e/cli/test_bootstrap_user_services + - run: mise run test:e2e e2e/cli/test_dotfiles_watch + - run: mise run test:e2e e2e/cli/test_dotfiles_watch_throttle lint: runs-on: ${{ !inputs.trusted && 'ubuntu-latest' || 'namespace-profile-endev-linux-amd64-large;overrides.cache-tag=cache' }} diff --git a/Cargo.lock b/Cargo.lock index 7b95d9f8950..30dfa2864d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3436,6 +3436,15 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "file-id" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fc6a637b6dc58414714eddd9170ff187ecb0933d4c7024d1abbd23a3cc26e9" +dependencies = [ + "windows-sys 0.60.2", +] + [[package]] name = "file_url" version = "0.3.2" @@ -3663,6 +3672,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "fsio" version = "0.4.1" @@ -4291,7 +4309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c91d8cffac8849493a82233811bd02b2b183b8cf39bf704de0fa0841b737595" dependencies = [ "bstr", - "hashbrown 0.15.5", + "hashbrown 0.17.1", ] [[package]] @@ -5629,6 +5647,26 @@ dependencies = [ "rustversion", ] +[[package]] +name = "inotify" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cc00ea907cab49550b7da656f80ebb97be1b997d931fbcd28d39734e17ce592" +dependencies = [ + "bitflags 2.13.1", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.1.4" @@ -5977,6 +6015,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "kqueue" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d763e5b24120b4ddf50de6c92308156765aabfbbccebf401da7cff2d70a41ea" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags 2.13.1", + "libc", +] + [[package]] name = "landlock" version = "0.4.7" @@ -6373,6 +6431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] @@ -6455,6 +6514,8 @@ dependencies = [ "netrc-rs", "nix 0.31.3", "nodejs-semver", + "notify", + "notify-debouncer-full", "nucleo-matcher", "num_cpus", "once_cell", @@ -6803,6 +6864,46 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" +[[package]] +name = "notify" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags 2.13.1", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-debouncer-full" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c02b49179cfebc9932238d04d6079912d26de0379328872846118a0fa0dbb302" +dependencies = [ + "file-id", + "log", + "notify", + "notify-types", + "walkdir", +] + +[[package]] +name = "notify-types" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" +dependencies = [ + "bitflags 2.13.1", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -7308,7 +7409,7 @@ dependencies = [ "aes-gcm", "aes-kw", "argon2", - "base64 0.21.7", + "base64 0.22.1", "bitfields", "block-padding 0.3.3", "blowfish", diff --git a/Cargo.toml b/Cargo.toml index d68b40717f6..be1a3e4a200 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,6 +152,10 @@ miette = { version = "7", features = ["fancy"] } netrc-rs = "0.1" nodejs-semver = "5" nucleo-matcher = "0.3" +notify = { version = "8", default-features = false, features = [ + "macos_fsevent", +] } +notify-debouncer-full = { version = "0.7", default-features = false } num_cpus = "1" once_cell = "1" packslip = { version = "0.3", default-features = false } diff --git a/docs/.vitepress/cli_commands.ts b/docs/.vitepress/cli_commands.ts index 52591d096ac..d4e69b54334 100644 --- a/docs/.vitepress/cli_commands.ts +++ b/docs/.vitepress/cli_commands.ts @@ -86,6 +86,9 @@ export const commands: { [key: string]: Command } = { edit: { hide: false, }, + exclude: { + hide: false, + }, history: { hide: false, subcommands: { @@ -103,6 +106,9 @@ export const commands: { [key: string]: Command } = { }, }, }, + include: { + hide: false, + }, paths: { hide: false, }, @@ -127,6 +133,9 @@ export const commands: { [key: string]: Command } = { untrack: { hide: false, }, + watch: { + hide: false, + }, }, }, files: { diff --git a/docs/cli/bootstrap/dotfiles.md b/docs/cli/bootstrap/dotfiles.md index 7287c878a2b..e7a536bec24 100644 --- a/docs/cli/bootstrap/dotfiles.md +++ b/docs/cli/bootstrap/dotfiles.md @@ -16,8 +16,10 @@ Manage dotfiles from `[dotfiles]` - [`mise bootstrap dotfiles apply [FLAGS] [TARGET]…`](/cli/bootstrap/dotfiles/apply.md) - [`mise bootstrap dotfiles diff [TARGET]…`](/cli/bootstrap/dotfiles/diff.md) - [`mise bootstrap dotfiles edit [FLAGS] `](/cli/bootstrap/dotfiles/edit.md) +- [`mise bootstrap dotfiles exclude `](/cli/bootstrap/dotfiles/exclude.md) - [`mise bootstrap dotfiles history [FLAGS] [SUBCOMMAND]`](/cli/bootstrap/dotfiles/history.md) -- [`mise bootstrap dotfiles paths [-J --json] [--preview ]`](/cli/bootstrap/dotfiles/paths.md) +- [`mise bootstrap dotfiles include `](/cli/bootstrap/dotfiles/include.md) +- [`mise bootstrap dotfiles paths [FLAGS]`](/cli/bootstrap/dotfiles/paths.md) - [`mise bootstrap dotfiles rollback [FLAGS] [PATH]…`](/cli/bootstrap/dotfiles/rollback.md) - [`mise bootstrap dotfiles save [FLAGS] [PATH]…`](/cli/bootstrap/dotfiles/save.md) - [`mise bootstrap dotfiles status [-J --json] [--missing] [TARGET]…`](/cli/bootstrap/dotfiles/status.md) @@ -25,3 +27,4 @@ Manage dotfiles from `[dotfiles]` - [`mise bootstrap dotfiles unapply [FLAGS] [TARGET]…`](/cli/bootstrap/dotfiles/unapply.md) - [`mise bootstrap dotfiles undo [-n --dry-run] [-y --yes] [REF]`](/cli/bootstrap/dotfiles/undo.md) - [`mise bootstrap dotfiles untrack …`](/cli/bootstrap/dotfiles/untrack.md) +- [`mise bootstrap dotfiles watch [--once] [-J --json]`](/cli/bootstrap/dotfiles/watch.md) diff --git a/docs/cli/bootstrap/dotfiles/exclude.md b/docs/cli/bootstrap/dotfiles/exclude.md new file mode 100644 index 00000000000..949372fa99d --- /dev/null +++ b/docs/cli/bootstrap/dotfiles/exclude.md @@ -0,0 +1,19 @@ + +# `mise bootstrap dotfiles exclude` + +- **Usage:** `mise bootstrap dotfiles exclude ` +- **Effect:** modifies state +- **Source code:** [`src/cli/dotfiles/exclude.rs`](https://github.com/jdx/mise/blob/main/src/cli/dotfiles/exclude.rs) + +Never capture paths matching a glob + +Adds the glob to `[history] exclude` in the global config. Use it for +logs, caches, databases, and constantly rewritten application state; a +file that genuinely holds configuration but changes constantly is +better tracked with `--no-autosave` and saved explicitly. + +## Arguments +- **``** — A glob such as `~/.config/hypr/plugins/**` + +## Flags +- **`-h --help`** — Print help diff --git a/docs/cli/bootstrap/dotfiles/include.md b/docs/cli/bootstrap/dotfiles/include.md new file mode 100644 index 00000000000..c3b29763ef8 --- /dev/null +++ b/docs/cli/bootstrap/dotfiles/include.md @@ -0,0 +1,16 @@ + +# `mise bootstrap dotfiles include` + +- **Usage:** `mise bootstrap dotfiles include ` +- **Effect:** modifies state +- **Source code:** [`src/cli/dotfiles/exclude.rs`](https://github.com/jdx/mise/blob/main/src/cli/dotfiles/exclude.rs) + +Capture paths matching a glob again + +Removes the glob from `[history] exclude` in the global config. + +## Arguments +- **``** — The glob as written by `exclude` + +## Flags +- **`-h --help`** — Print help diff --git a/docs/cli/bootstrap/dotfiles/paths.md b/docs/cli/bootstrap/dotfiles/paths.md index f9a478d8df6..66bbbcde6c5 100644 --- a/docs/cli/bootstrap/dotfiles/paths.md +++ b/docs/cli/bootstrap/dotfiles/paths.md @@ -1,7 +1,7 @@ # `mise bootstrap dotfiles paths` -- **Usage:** `mise bootstrap dotfiles paths [-J --json] [--preview ]` +- **Usage:** `mise bootstrap dotfiles paths [FLAGS]` - **Effect:** read-only - **Source code:** [`src/cli/dotfiles/paths.rs`](https://github.com/jdx/mise/blob/main/src/cli/dotfiles/paths.rs) @@ -15,4 +15,5 @@ enrollment is never mistaken for protection. ## Flags - **`-J --json`** — Output in JSON format - **`--preview `** — Show what tracking this path would capture +- **`--noisy`** — List the paths the watcher found changing constantly - **`-h --help`** — Print help diff --git a/docs/cli/bootstrap/dotfiles/watch.md b/docs/cli/bootstrap/dotfiles/watch.md new file mode 100644 index 00000000000..d35173ec163 --- /dev/null +++ b/docs/cli/bootstrap/dotfiles/watch.md @@ -0,0 +1,42 @@ + +# `mise bootstrap dotfiles watch` + +- **Usage:** `mise bootstrap dotfiles watch [--once] [-J --json]` +- **Effect:** modifies state +- **Source code:** [`src/cli/dotfiles/watch.rs`](https://github.com/jdx/mise/blob/main/src/cli/dotfiles/watch.rs) + +Save tracked files as they change + +Runs in the foreground: installs filesystem watches for every autosaved +tracked entry, saves a checkpoint once a changed file has been quiet for +`history.watch.debounce` (a file that keeps changing never delays the +others; `history.watch.max_interval` saves it regardless), and +reconciles the whole set at startup, every `history.watch.reconcile`, +and when the configuration changes. Manual-save entries are never +watched. + +The `history-watch` built-in service runs this for you: + +``` +[bootstrap.services.mise-history] +builtin = "history-watch" +``` + +Exit codes: 0 when history is disabled or another watcher already runs; +1 when git is unusable, the store cannot open, or no watch can be +installed. A capture that fails is retried with backoff and never drops +the pending changes; one that would overlap another history operation +is deferred. + +## Flags +- **`--once`** — Reconcile once and exit (for timers and cron) +- **`-J --json`** — One JSON object per line instead of log lines +- **`-h --help`** — Print help + +Examples: + +``` +mise bootstrap dotfiles watch +mise bootstrap dotfiles watch --once # one reconcile, for a timer +mise bootstrap dotfiles watch --json +``` diff --git a/docs/cli/index.md b/docs/cli/index.md index d2a3f1ad966..68c5b047469 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -67,12 +67,14 @@ - [`mise bootstrap dotfiles apply [FLAGS] [TARGET]…`](/cli/bootstrap/dotfiles/apply.md) - [`mise bootstrap dotfiles diff [TARGET]…`](/cli/bootstrap/dotfiles/diff.md) - [`mise bootstrap dotfiles edit [FLAGS] `](/cli/bootstrap/dotfiles/edit.md) +- [`mise bootstrap dotfiles exclude `](/cli/bootstrap/dotfiles/exclude.md) - [`mise bootstrap dotfiles history [FLAGS] [SUBCOMMAND]`](/cli/bootstrap/dotfiles/history.md) - [`mise bootstrap dotfiles history describe `](/cli/bootstrap/dotfiles/history/describe.md) - [`mise bootstrap dotfiles history diff [FLAGS] [A] [B]`](/cli/bootstrap/dotfiles/history/diff.md) - [`mise bootstrap dotfiles history ls [FLAGS]`](/cli/bootstrap/dotfiles/history/ls.md) - [`mise bootstrap dotfiles history show [FLAGS] [REF]`](/cli/bootstrap/dotfiles/history/show.md) -- [`mise bootstrap dotfiles paths [-J --json] [--preview ]`](/cli/bootstrap/dotfiles/paths.md) +- [`mise bootstrap dotfiles include `](/cli/bootstrap/dotfiles/include.md) +- [`mise bootstrap dotfiles paths [FLAGS]`](/cli/bootstrap/dotfiles/paths.md) - [`mise bootstrap dotfiles rollback [FLAGS] [PATH]…`](/cli/bootstrap/dotfiles/rollback.md) - [`mise bootstrap dotfiles save [FLAGS] [PATH]…`](/cli/bootstrap/dotfiles/save.md) - [`mise bootstrap dotfiles status [-J --json] [--missing] [TARGET]…`](/cli/bootstrap/dotfiles/status.md) @@ -80,6 +82,7 @@ - [`mise bootstrap dotfiles unapply [FLAGS] [TARGET]…`](/cli/bootstrap/dotfiles/unapply.md) - [`mise bootstrap dotfiles undo [-n --dry-run] [-y --yes] [REF]`](/cli/bootstrap/dotfiles/undo.md) - [`mise bootstrap dotfiles untrack …`](/cli/bootstrap/dotfiles/untrack.md) +- [`mise bootstrap dotfiles watch [--once] [-J --json]`](/cli/bootstrap/dotfiles/watch.md) - [`mise bootstrap files `](/cli/bootstrap/files.md) - [`mise bootstrap files apply [FLAGS]`](/cli/bootstrap/files/apply.md) - [`mise bootstrap files status [FLAGS]`](/cli/bootstrap/files/status.md) diff --git a/docs/history.md b/docs/history.md index 992a5b79e11..06ac4f32741 100644 --- a/docs/history.md +++ b/docs/history.md @@ -130,6 +130,111 @@ forced, and an empty directory it replaced. It refuses when that checkpoint was pruned. Undoing an undo re-applies the operation; an undo that changed nothing does not count as having reversed it. +## Automatic saves + +`mise bootstrap dotfiles watch` saves tracked files as they change, whatever +wrote them: an editor, a script, an agent, a distro update, or mise itself. +Declare it once as the built-in user service and `mise bootstrap` installs +and starts it on every platform (a systemd user unit, a LaunchAgent, or a +Scheduled Task): + +```toml +[bootstrap.services.mise-history] +builtin = "history-watch" +``` + +```sh +mise bootstrap services apply # or the full `mise bootstrap` +mise bootstrap dotfiles status # watcher: running +``` + +The watcher installs filesystem watches for every autosaved entry (a tracked +directory recursively, a tracked file through its parent, a path that does +not exist yet through its nearest existing ancestor). Manual-save entries +(`autosave = false`) are never watched. + +### Adaptive scheduling + +Every file is scheduled on its own. An ordinary edit is saved once the file +has been quiet for `history.watch.debounce` (2s). A file that is rewritten +constantly is not saved on every change: when a save follows the previous +one without the file ever settling, that file's own interval doubles, up to +`history.watch.max_interval` (24h). It is still saved periodically at that +interval for as long as it keeps changing; nothing is ever excluded or +switched to manual saving automatically, and a checkpoint another file +triggers carries the throttled file's last saved version, not its live +content, so a whole-set reconciliation never defeats the throttling. As soon +as the file stops changing its final state is captured promptly (after a +fraction of its interval, at most five minutes), and a sustained quiet +period (four intervals, at least five minutes) resets it to the base interval. +A busy file never delays an ordinary one. Explicit saves and the protective +checkpoints before a bootstrap, rollback, or undo always read every file +live. + +The thresholds are fixed: an interval doubles when a file changed again +within its settle time of the previous save and at least two changes +arrived since. A person saving from an editor every few seconds leaves gaps +longer than the settle time, so ordinary editing is never stretched. The +schedule is persisted (`watch-schedule.json` in the history store) with +each throttled file's last save and pending changes, so a restart of the +service continues where it stopped: the startup capture holds a throttled +file at its saved version until its next save is due, and a file rewritten +while the service was down is pending, not saved early. Editing +`history.watch.debounce`, `history.watch.max_interval`, or +`history.watch.reconcile` in the global configuration takes effect while +the service runs. + +Constantly rewritten application state, logs, caches, and databases are +better excluded, and a file that genuinely holds configuration but changes +constantly can be tracked with `autosave = false` and saved explicitly: + +```sh +mise bootstrap dotfiles paths --noisy # what is throttled right now +mise bootstrap dotfiles exclude '~/.config/hypr/plugins/**' # [history] exclude +mise bootstrap dotfiles include '~/.config/hypr/plugins/**' +mise bootstrap dotfiles track ~/.config/app/state.json --no-autosave +mise bootstrap dotfiles save ~/.config/app/state.json +``` + +### Reconciliation and failures + +The whole tracked set is reconciled at startup, every +`history.watch.reconcile` (10m; `0` disables), when the configuration +changes (an edit to `~/.config/mise/*.toml` or `conf.d/` reloads the +declarations and replans the watches; `history.enabled = false` stops the +watcher), and on shutdown, so an edit no watch reported is still saved. +`mise bootstrap dotfiles watch --once` runs one reconcile and exits, for a +timer or cron instead of the service. + +A capture that fails is retried with backoff (1s to 5min) and never drops +the pending changes; one that would overlap another history operation (a +running bootstrap, rollback, or undo) is deferred and retried until that +operation finishes, whether or not any other save is due. The shutdown +capture waits a moment for a running operation and says what stays unsaved +if it cannot. `mise bootstrap dotfiles watch --once` exits 1 when nothing +could be saved (deferred or failed), so a timer notices. One watcher runs +per store: a second one exits 0 immediately. +`--json` prints one object per line (`started`, `captured`, `unchanged`, +`deferred`, `replan`, `throttled`, `settled`, `degraded`, `error`, +`stopped`). + +### Health + +The watcher never notifies you. It persists its health (`health.json` in +the history store) and two commands read it, without starting a sync, +applying anything, or prompting: + +- `mise doctor` prints a concise `dotfiles` section: a watcher that is + declared but not running (with the command that starts it), repeated + capture failures or an unusable store, and heavily throttled files. A + throttled file is informational, not a warning. Health older than a few + reconcile intervals is reported as stale rather than current. +- `mise bootstrap dotfiles status` prints the detail: the watcher state + (`running`, `declared but not running`, `not declared`), the last capture + and reconcile, the last failure, and for every throttled file its + effective interval, last save, and unsaved changes (changes seen since the + last save, kept current as they happen). + ## What is tracked `mise bootstrap dotfiles paths` lists every entry with its mode, policies, the file that diff --git a/e2e-win/dotfiles_watch.Tests.ps1 b/e2e-win/dotfiles_watch.Tests.ps1 new file mode 100644 index 00000000000..af30cf08194 --- /dev/null +++ b/e2e-win/dotfiles_watch.Tests.ps1 @@ -0,0 +1,54 @@ +Describe 'history watch' { + BeforeAll { + $script:OriginalDir = Get-Location + Set-Location TestDrive: + + $script:OriginalTrusted = [Environment]::GetEnvironmentVariable('MISE_TRUSTED_CONFIG_PATHS', 'Process') + $env:MISE_TRUSTED_CONFIG_PATHS = $TestDrive + $script:OriginalConfigDir = [Environment]::GetEnvironmentVariable('MISE_CONFIG_DIR', 'Process') + $script:OriginalStateDir = [Environment]::GetEnvironmentVariable('MISE_STATE_DIR', 'Process') + $env:MISE_CONFIG_DIR = Join-Path $TestDrive 'config' + $env:MISE_STATE_DIR = Join-Path $TestDrive 'state' + New-Item -ItemType Directory -Force -Path $env:MISE_CONFIG_DIR | Out-Null + $script:Tracked = Join-Path $TestDrive 'tracked' + New-Item -ItemType Directory -Force -Path $script:Tracked | Out-Null + 'one' | Out-File -FilePath (Join-Path $script:Tracked 'file.txt') -Encoding utf8NoBOM + } + + AfterAll { + Set-Location $script:OriginalDir + foreach ($pair in @( + @('MISE_TRUSTED_CONFIG_PATHS', $script:OriginalTrusted), + @('MISE_CONFIG_DIR', $script:OriginalConfigDir), + @('MISE_STATE_DIR', $script:OriginalStateDir))) { + if ($null -eq $pair[1]) { + Remove-Item ("Env:" + $pair[0]) -ErrorAction Ignore + } else { + [Environment]::SetEnvironmentVariable($pair[0], $pair[1], 'Process') + } + } + } + + It 'reconciles once and reports capture health' { + $tracked = $script:Tracked -replace '\\', '/' + mise bootstrap dotfiles track $tracked 2>&1 | Out-String | Out-Null + $LASTEXITCODE | Should -Be 0 + + $status = mise bootstrap dotfiles status --json | Out-String | ConvertFrom-Json + $status.history.watcher | Should -Be 'not-declared' + + 'two' | Out-File -FilePath (Join-Path $script:Tracked 'file.txt') -Encoding utf8NoBOM + mise bootstrap dotfiles watch --once 2>&1 | Out-String | Out-Null + $LASTEXITCODE | Should -Be 0 + $entries = mise bootstrap dotfiles history --json | Out-String | ConvertFrom-Json + $entries.Count | Should -Be 2 + $entries[0].trigger | Should -Be 'edit' + + @" +[bootstrap.services.mise-history] +builtin = "history-watch" +"@ | Out-File -FilePath (Join-Path $env:MISE_CONFIG_DIR 'config.toml') -Encoding utf8NoBOM + $status = mise bootstrap dotfiles status --json | Out-String | ConvertFrom-Json + $status.history.watcher | Should -Be 'declared-not-running' + } +} diff --git a/e2e/cli/test_dotfiles_watch b/e2e/cli/test_dotfiles_watch new file mode 100644 index 00000000000..15da93b6f10 --- /dev/null +++ b/e2e/cli/test_dotfiles_watch @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# `mise bootstrap dotfiles watch`: edits to tracked files are saved automatically once +# they are quiet, configuration changes replan the watches, one watcher runs +# per store, and `--once` reconciles for timers. +# shellcheck disable=SC2016 + +require_cmd git + +export MISE_HISTORY_WATCH_DEBOUNCE=1s +export MISE_HISTORY_WATCH_MAX_INTERVAL=5s + +count="mise bootstrap dotfiles history --json -n 0 | jq length" + +# waits until the checkpoint count reaches $1 (or fails after ~10s) +wait_for_count() { + for _ in $(seq 1 50); do + if [[ "$(eval "$count")" -ge $1 ]]; then + return 0 + fi + sleep 0.2 + done + echo "expected at least $1 checkpoints, found $(eval "$count")" >&2 + [[ -f watch.log ]] && cat watch.log >&2 + return 1 +} + +mkdir -p ~/.config/hypr +echo 'monitor=DP-1' >~/.config/hypr/monitors.lua +assert_succeed "mise bootstrap dotfiles track ~/.config/hypr" +assert "$count" "1" +assert_contains "mise bootstrap dotfiles status" "automatic capture: not declared" + +# a declared but not running watcher is reported with the next command +cat <<'EOF' >>"$MISE_CONFIG_DIR/config.toml" + +[bootstrap.services.mise-history] +builtin = "history-watch" +EOF +assert_contains "mise bootstrap dotfiles status" "declared but not running" +assert_contains "mise bootstrap dotfiles status" "mise bootstrap services apply" +assert "mise bootstrap dotfiles status --json | jq -r .history.watcher" "declared-not-running" + +# --once reconciles and exits: an edit made while nothing watched is saved +echo 'monitor=DP-2' >~/.config/hypr/monitors.lua +assert_succeed "mise bootstrap dotfiles watch --once" +assert "$count" "2" +assert "mise bootstrap dotfiles history --json | jq -r '.[0].trigger'" "edit" +assert "mise bootstrap dotfiles history --json | jq -r '.[0].description'" "edited hypr/monitors.lua, mise/config.toml" + +# the foreground watcher saves an edit once the file is quiet +mise bootstrap dotfiles watch --json >watch.log 2>&1 & +watcher=$! +cleanup() { kill "$watcher" 2>/dev/null || true; } +trap cleanup EXIT +for _ in $(seq 1 50); do + grep -q '"event":"started"' watch.log && break + sleep 0.2 +done +assert_contains "cat watch.log" '"event":"started"' +assert "mise bootstrap dotfiles status --json | jq -r .history.watcher" "running" +assert_contains "mise bootstrap dotfiles status" "automatic capture: running" + +# one watcher per store: a second one exits 0 immediately +assert_contains "mise bootstrap dotfiles watch --json" '"event":"already-running"' + +echo 'monitor=DP-3' >~/.config/hypr/monitors.lua +echo 'bind = SUPER, Q' >~/.config/hypr/bindings.lua +wait_for_count 3 +assert "mise bootstrap dotfiles history --json | jq -r '.[0].description'" "edited hypr/monitors.lua; added hypr/bindings.lua" +assert_contains "cat watch.log" '"event":"captured"' + +# a configuration change replans: an excluded file stops being captured +assert_succeed "mise bootstrap dotfiles exclude '~/.config/hypr/bindings.lua'" +wait_for_count 4 +assert_contains "cat watch.log" '"event":"replan"' +assert "mise bootstrap dotfiles history --json | jq -r '.[0].description'" "edited mise/config.toml; removed hypr/bindings.lua" +echo 'bind = SUPER, W' >~/.config/hypr/bindings.lua +echo 'monitor=DP-4' >~/.config/hypr/monitors.lua +wait_for_count 5 +assert "mise bootstrap dotfiles history --json | jq -r '.[0].description'" "edited hypr/monitors.lua" +assert_succeed "mise bootstrap dotfiles include '~/.config/hypr/bindings.lua'" +wait_for_count 6 + +# a running history operation defers the watcher instead of racing it: the +# rollback records its own pair and the watcher saves nothing extra +n="$(eval "$count")" +assert_succeed "mise bootstrap dotfiles rollback ~/.config/hypr/monitors.lua --yes" +sleep 2.5 +assert "cat ~/.config/hypr/monitors.lua" "monitor=DP-3" +assert "$count" "$((n + 2))" + +# stopping saves what is still pending +echo 'monitor=DP-5' >~/.config/hypr/monitors.lua +sleep 0.3 +kill -TERM "$watcher" +wait "$watcher" || true +assert_contains "cat watch.log" '"event":"stopped"' +assert "mise bootstrap dotfiles history --json | jq -r '.[0].description'" "edited hypr/monitors.lua" +assert_contains "mise bootstrap dotfiles status" "declared but not running" + +# the watcher is a no-op when history is disabled, and fails clearly when +# nothing can be watched +assert_contains "MISE_HISTORY_ENABLED=0 mise bootstrap dotfiles watch --json" '"event":"disabled"' +assert_succeed "MISE_HISTORY_ENABLED=0 mise bootstrap dotfiles watch" diff --git a/e2e/cli/test_dotfiles_watch_throttle b/e2e/cli/test_dotfiles_watch_throttle new file mode 100644 index 00000000000..1395d9446ff --- /dev/null +++ b/e2e/cli/test_dotfiles_watch_throttle @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# Adaptive autosaving: a constantly rewritten file is saved ever more rarely +# (never excluded, never switched to manual saving), never delays an ordinary +# edit, is captured promptly once it settles, keeps its throttling across a +# watcher restart, and is reported by `mise doctor` and `dotfiles status` +# without either of them changing anything. +# shellcheck disable=SC2016 + +require_cmd git + +count="mise bootstrap dotfiles history --json -n 0 | jq length" +throttled="mise bootstrap dotfiles status --json | jq -r '.history.health.throttled[0].interval_secs // 0'" +repo="$MISE_STATE_DIR/history/repo.git" +# the content of a path inside a checkpoint's snapshot (a script: assertions +# run in a subshell where shell functions are not available) +snapshot_of="$PWD/snapshot_of" +cat <"$snapshot_of" +#!/usr/bin/env bash +git --git-dir="$repo" show "refs/checkpoints/\$(mise bootstrap dotfiles history show "\$1" --json | jq -r .uuid):snapshot/home/\$2" +EOF2 +chmod +x "$snapshot_of" + +wait_until() { + for _ in $(seq 1 100); do + if eval "$1" >/dev/null 2>&1; then + return 0 + fi + sleep 0.2 + done + echo "timed out waiting for: $1" >&2 + [[ -f watch.log ]] && cat watch.log >&2 + return 1 +} + +mkdir -p ~/.config/hypr +echo 'monitor=DP-1' >~/.config/hypr/monitors.lua +echo '{"n":0}' >~/.config/hypr/state.json +assert_succeed "mise bootstrap dotfiles track ~/.config/hypr" +# the timing settings live in the configuration so a change to them can be +# observed while the watcher runs; reconciliation is off, so only events and +# retries save anything +cat <<'EOF' >>"$MISE_CONFIG_DIR/config.toml" + +[settings] +history.watch.debounce = "2s" +history.watch.max_interval = "8s" +history.watch.reconcile = "0" + +[bootstrap.services.mise-history] +builtin = "history-watch" +EOF + +# doctor reports the declared-but-not-running watcher without touching anything +n="$(eval "$count")" +assert_contains "mise doctor 2>&1 || true" "the history watcher is declared but not running" +assert_contains "mise doctor 2>&1 || true" "mise bootstrap services apply" +assert "mise doctor --json 2>/dev/null | jq -r .dotfiles.watcher" "declared but not running" +assert "$count" "$n" + +mise bootstrap dotfiles watch --json >watch.log 2>&1 & +watcher=$! +churn="" +cleanup() { + kill "$churn" 2>/dev/null || true + kill "$watcher" 2>/dev/null || true +} +trap cleanup EXIT +wait_until "grep -q '\"event\":\"started\"' watch.log" + +# a loop rewrites state.json ten times a second +( + i=0 + while :; do + i=$((i + 1)) + echo "{\"n\":$i}" >~/.config/hypr/state.json + sleep 0.1 + done +) & +churn=$! + +# the churning file is throttled: its interval grows past the base +wait_until "[[ \$($throttled) -ge 8 ]]" +assert_contains "cat watch.log" '"event":"throttled"' +assert_contains "mise bootstrap dotfiles paths --noisy" "state.json" +assert_contains "mise bootstrap dotfiles status" "throttled: ~/.config/hypr/state.json changes constantly" +assert_contains "mise doctor 2>&1 || true" "state.json changes constantly" +assert_not_contains "mise doctor 2>&1 || true" "warning" +# never excluded, never switched to manual saving +assert_not_contains "cat $MISE_CONFIG_DIR/config.toml" "exclude" +assert_not_contains "cat $MISE_CONFIG_DIR/config.toml" "autosave" + +# an ordinary edit is still saved promptly while the churn goes on, and the +# checkpoint it makes does not carry the churning file's live content (the +# edit lands right after a save of the churning file, so its next save is a +# whole interval away) +captured="grep -c '\"event\":\"captured\"' watch.log" +n="$(eval "$captured")" +wait_until "[[ \$($captured) -gt $n ]]" +echo 'monitor=DP-2' >~/.config/hypr/monitors.lua +wait_until "mise bootstrap dotfiles history --json | jq -e '.[0].description | test(\"monitors.lua\")'" +saved_n="$(mise bootstrap dotfiles history show latest --files 2>/dev/null | grep -c state.json || true)" +assert "echo $saved_n" "1" + +# unsaved changes to the throttled file are visible as they happen +wait_until "[[ \$(mise bootstrap dotfiles status --json | jq -r '.history.health.throttled[0].pending_changes // 0') -ge 1 ]]" + +# the throttling survives a restart of the watcher while the churn goes on: +# the startup capture holds the throttled file at its saved version instead +# of saving it early, and its pending changes are still reported +kill -TERM "$watcher" +wait "$watcher" || true +assert_contains "cat $MISE_STATE_DIR/history/watch-schedule.json" "state.json" +saved_before="$($snapshot_of latest .config/hypr/state.json)" +sleep 1.2 +mise bootstrap dotfiles watch --json >watch.log 2>&1 & +watcher=$! +wait_until "grep -q '\"event\":\"started\"' watch.log" +assert "$snapshot_of latest .config/hypr/state.json" "$saved_before" +assert "mise bootstrap dotfiles status --json | jq -r '.history.health.throttled | length'" "1" +wait_until "[[ \$(mise bootstrap dotfiles status --json | jq -r '.history.health.throttled[0].pending_changes // 0') -ge 1 ]]" + +# excluding the churning file while it is throttled takes it out of the +# schedule at once: no later capture holds it or carries its old version +assert_succeed "mise bootstrap dotfiles exclude '~/.config/hypr/state.json'" +wait_until "grep -q 'configuration changed' watch.log" +wait_until "[[ \$(mise bootstrap dotfiles status --json | jq -r '.history.health.throttled | length') -eq 0 ]]" +assert_not_contains "cat $MISE_STATE_DIR/history/watch-schedule.json" "state.json" +echo 'monitor=DP-9' >~/.config/hypr/monitors.lua +wait_until "mise bootstrap dotfiles history --json | jq -e '.[0].description | test(\"monitors.lua\")'" +assert_not_contains "mise bootstrap dotfiles history show latest --files" "state.json" +assert_succeed "mise bootstrap dotfiles include '~/.config/hypr/state.json'" +wait_until "[[ \$($throttled) -ge 2 ]]" + +# a timing setting changed while running takes effect: the interval is +# clamped to the new maximum +sed -i.bak 's/max_interval = "8s"/max_interval = "2s"/' "$MISE_CONFIG_DIR/config.toml" +wait_until "[[ \$($throttled) -le 2 ]]" + +# the churn stops: the final state is captured promptly +kill "$churn" +churn="" +sleep 0.3 +final="$(cat ~/.config/hypr/state.json)" +wait_until "mise bootstrap dotfiles history diff --path ~/.config/hypr/state.json --exit-code" +# and nothing else is saved once everything is quiet +n="$(eval "$count")" +sleep 3 +assert "$count" "$n" + +# the target of a symlink inside a tracked directory is watched too, even +# outside every declared root and with reconciliation off +echo 'outside' >~/outside.conf +ln -s ~/outside.conf ~/.config/hypr/link.conf +wait_until "grep -q 'a tracked path appeared' watch.log" +echo 'outside2' >~/outside.conf +wait_until "mise bootstrap dotfiles history --json | jq -e '.[0].description | test(\"outside.conf\")'" + +# a capture deferred by a running history operation is retried once the +# operation finished (nothing else could save it: reconciliation is off). +# The operation is an apply whose pre-dotfiles hook holds the operation lock +# for a while +cat <<'EOF' >>"$MISE_CONFIG_DIR/config.toml" + +[bootstrap.hooks.pre-dotfiles] +run = "date +%s%N >>$HOME/applying && sleep 4" +EOF +wait_until "grep -q 'configuration changed' watch.log" +mise bootstrap dotfiles apply --yes >apply.log 2>&1 & +applying=$! +sleep 1 +echo 'monitor=DP-3' >~/.config/hypr/monitors.lua +wait_until "grep -q '\"event\":\"deferred\"' watch.log" +wait "$applying" || true +wait_until "mise bootstrap dotfiles history --json | jq -e '.[0].description | test(\"monitors.lua\")'" + +kill -TERM "$watcher" +wait "$watcher" || true +assert "cat ~/.config/hypr/state.json" "$final" + +# `--once` reports that nothing was saved while an operation holds the lock +# (the hook's marker says the lock is held; a slow start would let `--once` +# run first) +mise bootstrap dotfiles apply --yes >apply.log 2>&1 & +applying=$! +wait_until 'test "$(wc -l <~/applying)" -ge 2' +assert_fail "mise bootstrap dotfiles watch --once --json 2>&1" '"reason":"deferred"' +wait "$applying" || true diff --git a/man/man1/mise.1 b/man/man1/mise.1 index 6b28dfe81e8..0d3fc148f8a 100644 --- a/man/man1/mise.1 +++ b/man/man1/mise.1 @@ -202,6 +202,9 @@ Show the changes needed to apply dotfiles from `[dotfiles]` \fBbootstrap dotfiles edit\fR Edit a managed dotfile source .TP +\fBbootstrap dotfiles exclude\fR +Never capture paths matching a glob +.TP \fBbootstrap dotfiles history\fR Browse the checkpoints of your dotfiles .TP @@ -220,6 +223,9 @@ List checkpoints, newest first \fBbootstrap dotfiles history show\fR Show one checkpoint: what triggered it, what changed, and its journal .TP +\fBbootstrap dotfiles include\fR +Capture paths matching a glob again +.TP \fBbootstrap dotfiles paths\fR Show what history tracks and under which policies .TP @@ -247,6 +253,9 @@ Reverse a rollback or undo \fBbootstrap dotfiles untrack\fR Stop tracking a file or directory .TP +\fBbootstrap dotfiles watch\fR +Save tracked files as they change +.TP \fBbootstrap files\fR Manage privileged files and directories from `[bootstrap.files]` and `[bootstrap.directories]` .TP @@ -1408,6 +1417,26 @@ Print help .TP \fB\fR Target to edit +.SH "MISE BOOTSTRAP DOTFILES EXCLUDE" +Never capture paths matching a glob + +Adds the glob to `[history] exclude` in the global config. Use it for +logs, caches, databases, and constantly rewritten application state; a +file that genuinely holds configuration but changes constantly is +better tracked with `\-\-no\-autosave` and saved explicitly. +.PP +\fBUsage:\fR mise bootstrap dotfiles exclude [OPTIONS] +.PP +\fBOptions:\fR +.PP +.TP +\fB\-h, \-\-help\fR +Print help +\fBArguments:\fR +.PP +.TP +\fB\fR +A glob such as `~/.config/hypr/plugins/**` .SH "MISE BOOTSTRAP DOTFILES HISTORY" Browse the checkpoints of your dotfiles @@ -1543,6 +1572,23 @@ Print help .TP \fB\fR Checkpoint id, `latest` (the default), `latest~N`, or a uuid prefix +.SH "MISE BOOTSTRAP DOTFILES INCLUDE" +Capture paths matching a glob again + +Removes the glob from `[history] exclude` in the global config. +.PP +\fBUsage:\fR mise bootstrap dotfiles include [OPTIONS] +.PP +\fBOptions:\fR +.PP +.TP +\fB\-h, \-\-help\fR +Print help +\fBArguments:\fR +.PP +.TP +\fB\fR +The glob as written by `exclude` .SH "MISE BOOTSTRAP DOTFILES PATHS" Show what history tracks and under which policies @@ -1562,6 +1608,9 @@ Output in JSON format \fB\-\-preview\fR \fI\fR Show what tracking this path would capture .TP +\fB\-\-noisy\fR +List the paths the watcher found changing constantly +.TP \fB\-h, \-\-help\fR Print help .SH "MISE BOOTSTRAP DOTFILES ROLLBACK" @@ -1777,6 +1826,41 @@ Print help .TP \fB\fR Paths to stop tracking +.SH "MISE BOOTSTRAP DOTFILES WATCH" +Save tracked files as they change + +Runs in the foreground: installs filesystem watches for every autosaved +tracked entry, saves a checkpoint once a changed file has been quiet for +`history.watch.debounce` (a file that keeps changing never delays the +others; `history.watch.max_interval` saves it regardless), and +reconciles the whole set at startup, every `history.watch.reconcile`, +and when the configuration changes. Manual\-save entries are never +watched. + +The `history\-watch` built\-in service runs this for you: + + [bootstrap.services.mise\-history] + builtin = "history\-watch" + +Exit codes: 0 when history is disabled or another watcher already runs; +1 when git is unusable, the store cannot open, or no watch can be +installed. A capture that fails is retried with backoff and never drops +the pending changes; one that would overlap another history operation +is deferred. +.PP +\fBUsage:\fR mise bootstrap dotfiles watch [OPTIONS] +.PP +\fBOptions:\fR +.PP +.TP +\fB\-\-once\fR +Reconcile once and exit (for timers and cron) +.TP +\fB\-J, \-\-json\fR +One JSON object per line instead of log lines +.TP +\fB\-h, \-\-help\fR +Print help .SH "MISE BOOTSTRAP FILES" Manage privileged files and directories from `[bootstrap.files]` and `[bootstrap.directories]` .PP diff --git a/mise.usage.kdl b/mise.usage.kdl index a8982ba7ee6..1f73f6d59a8 100644 --- a/mise.usage.kdl +++ b/mise.usage.kdl @@ -655,6 +655,18 @@ mise doesn't otherwise own. flag "-h --help" help="Print help" action=help builtin=#true arg help="Target to edit" } + cmd exclude help="Never capture paths matching a glob" effect=write { + long_help #""" +Never capture paths matching a glob + +Adds the glob to `[history] exclude` in the global config. Use it for +logs, caches, databases, and constantly rewritten application state; a +file that genuinely holds configuration but changes constantly is +better tracked with `--no-autosave` and saved explicitly. +"""# + flag "-h --help" help="Print help" action=help builtin=#true + arg help="A glob such as `~/.config/hypr/plugins/**`" + } cmd history help="Browse the checkpoints of your dotfiles" effect=read { long_help #""" Browse the checkpoints of your dotfiles @@ -725,6 +737,15 @@ against the one before it. With two, compares the two states. arg "[REF]" help="Checkpoint id, `latest` (the default), `latest~N`, or a uuid prefix" required=#false } } + cmd include help="Capture paths matching a glob again" effect=write { + long_help #""" +Capture paths matching a glob again + +Removes the glob from `[history] exclude` in the global config. +"""# + flag "-h --help" help="Print help" action=help builtin=#true + arg help="The glob as written by `exclude`" + } cmd paths help="Show what history tracks and under which policies" effect=read { long_help #""" Show what history tracks and under which policies @@ -738,6 +759,7 @@ enrollment is never mistaken for protection. flag --preview help="Show what tracking this path would capture" { arg } + flag --noisy help="List the paths the watcher found changing constantly" flag "-h --help" help="Print help" action=help builtin=#true } cmd rollback help="Return files to the version a checkpoint holds" effect=destructive { @@ -865,6 +887,34 @@ existing checkpoints are left exactly as they are. flag "-h --help" help="Print help" action=help builtin=#true arg … help="Paths to stop tracking" var=#true } + cmd watch help="Save tracked files as they change" effect=write { + long_help #""" +Save tracked files as they change + +Runs in the foreground: installs filesystem watches for every autosaved +tracked entry, saves a checkpoint once a changed file has been quiet for +`history.watch.debounce` (a file that keeps changing never delays the +others; `history.watch.max_interval` saves it regardless), and +reconciles the whole set at startup, every `history.watch.reconcile`, +and when the configuration changes. Manual-save entries are never +watched. + +The `history-watch` built-in service runs this for you: + + [bootstrap.services.mise-history] + builtin = "history-watch" + +Exit codes: 0 when history is disabled or another watcher already runs; +1 when git is unusable, the store cannot open, or no watch can be +installed. A capture that fails is retried with backoff and never drops +the pending changes; one that would overlap another history operation +is deferred. +"""# + after_long_help "\u{1b}[1m\u{1b}[4mExamples:\u{1b}[22m\u{1b}[24m\n\n $ \u{1b}[1mmise bootstrap dotfiles watch\u{1b}[22m\n $ \u{1b}[1mmise bootstrap dotfiles watch --once\u{1b}[22m # one reconcile, for a timer\n $ \u{1b}[1mmise bootstrap dotfiles watch --json\u{1b}[22m\n" + flag --once help="Reconcile once and exit (for timers and cron)" + flag "-J --json" help="One JSON object per line instead of log lines" + flag "-h --help" help="Print help" action=help builtin=#true + } } cmd files subcommand_required=#true help="Manage privileged files and directories from `[bootstrap.files]` and `[bootstrap.directories]`" effect=read { flag "-h --help" help="Print help" action=help builtin=#true diff --git a/schema/mise.json b/schema/mise.json index 70d1e4d6719..1996821a4dd 100644 --- a/schema/mise.json +++ b/schema/mise.json @@ -1142,6 +1142,27 @@ "minimum": 0 } } + }, + "watch": { + "type": "object", + "unevaluatedProperties": false, + "properties": { + "debounce": { + "default": "2s", + "description": "How long a changed file must stay quiet before the history watcher saves it (the base autosave interval). A file that keeps changing is stretched on its own and never delays the others.", + "type": "string" + }, + "max_interval": { + "default": "24h", + "description": "The longest autosave interval the history watcher stretches a constantly changing file to. Sustained churn doubles a file's own interval up to this; a settled file is saved promptly again, and a sustained quiet period resets it.", + "type": "string" + }, + "reconcile": { + "default": "10m", + "description": "How often the history watcher rescans the whole tracked set for changes its watches missed. `0` disables periodic reconciliation (startup and configuration changes still reconcile).", + "type": "string" + } + } } } }, diff --git a/settings.toml b/settings.toml index f666ad7d603..b504b70afb7 100644 --- a/settings.toml +++ b/settings.toml @@ -1274,6 +1274,32 @@ env = "MISE_HISTORY_KEEP_COUNT" rust_type = "usize" type = "Integer" +[history.watch.debounce] +default = "2s" +description = "How long a changed file must stay quiet before the history watcher saves it (the base autosave interval). A file that keeps changing is stretched on its own and never delays the others." +env = "MISE_HISTORY_WATCH_DEBOUNCE" +type = "Duration" + +[history.watch.max_interval] +default = "24h" +description = "The longest autosave interval the history watcher stretches a constantly changing file to. Sustained churn doubles a file's own interval up to this; a settled file is saved promptly again, and a sustained quiet period resets it." +docs = """ +Ordinary edits are saved after `history.watch.debounce`. A file that keeps being rewritten is +still saved periodically, ever more rarely, up to this interval; other files are never delayed +by it. Constantly rewritten application state, logs, caches, and databases are better excluded +(`mise bootstrap dotfiles exclude`); a file that genuinely holds configuration but changes +constantly can be tracked with `autosave = false` and saved explicitly. Nothing is excluded or +switched to manual saving automatically. +""" +env = "MISE_HISTORY_WATCH_MAX_INTERVAL" +type = "Duration" + +[history.watch.reconcile] +default = "10m" +description = "How often the history watcher rescans the whole tracked set for changes its watches missed. `0` disables periodic reconciliation (startup and configuration changes still reconcile)." +env = "MISE_HISTORY_WATCH_RECONCILE" +type = "Duration" + [hook_env.cache_ttl] default = "0s" description = "Cache hook-env directory checks for this duration. Useful for slow filesystems like NFS." diff --git a/src/cli/bootstrap.rs b/src/cli/bootstrap.rs index 38a1f745f86..899172f6eb9 100644 --- a/src/cli/bootstrap.rs +++ b/src/cli/bootstrap.rs @@ -10,9 +10,9 @@ use serde::Serialize; use serde_json::{Value, json}; use super::dotfiles::{ - DotfilesAdd, DotfilesApply, DotfilesDiff, DotfilesEdit, DotfilesHistory, DotfilesPaths, - DotfilesRollback, DotfilesSave, DotfilesStatus, DotfilesTrack, DotfilesUnapply, DotfilesUndo, - DotfilesUntrack, + DotfilesAdd, DotfilesApply, DotfilesDiff, DotfilesEdit, DotfilesExclude, DotfilesHistory, + DotfilesInclude, DotfilesPaths, DotfilesRollback, DotfilesSave, DotfilesStatus, DotfilesTrack, + DotfilesUnapply, DotfilesUndo, DotfilesUntrack, DotfilesWatch, }; use super::install::Install; use super::plugins::install::install_plugin; @@ -829,7 +829,9 @@ enum BootstrapDotfilesCommands { Apply(BootstrapDotfilesApply), Diff(DotfilesDiff), Edit(DotfilesEdit), + Exclude(DotfilesExclude), History(DotfilesHistory), + Include(DotfilesInclude), Paths(DotfilesPaths), Rollback(DotfilesRollback), Save(DotfilesSave), @@ -838,6 +840,7 @@ enum BootstrapDotfilesCommands { Unapply(DotfilesUnapply), Undo(DotfilesUndo), Untrack(DotfilesUntrack), + Watch(DotfilesWatch), } /// Apply dotfiles from `[dotfiles]` @@ -4064,7 +4067,9 @@ impl BootstrapDotfiles { BootstrapDotfilesCommands::Apply(cmd) => cmd.run().await, BootstrapDotfilesCommands::Diff(cmd) => cmd.run().await, BootstrapDotfilesCommands::Edit(cmd) => cmd.run().await, + BootstrapDotfilesCommands::Exclude(cmd) => cmd.run().await, BootstrapDotfilesCommands::History(cmd) => cmd.run().await, + BootstrapDotfilesCommands::Include(cmd) => cmd.run().await, BootstrapDotfilesCommands::Paths(cmd) => cmd.run().await, BootstrapDotfilesCommands::Rollback(cmd) => cmd.run().await, BootstrapDotfilesCommands::Save(cmd) => cmd.run().await, @@ -4073,6 +4078,7 @@ impl BootstrapDotfiles { BootstrapDotfilesCommands::Unapply(cmd) => cmd.run().await, BootstrapDotfilesCommands::Undo(cmd) => cmd.run().await, BootstrapDotfilesCommands::Untrack(cmd) => cmd.run().await, + BootstrapDotfilesCommands::Watch(cmd) => cmd.run().await, } } } diff --git a/src/cli/command_effects.rs b/src/cli/command_effects.rs index b3b05adbda1..9560842c031 100644 --- a/src/cli/command_effects.rs +++ b/src/cli/command_effects.rs @@ -82,11 +82,13 @@ pub(super) const EFFECTS: &[(&str, SpecCommandEffect)] = &[ ("bootstrap dotfiles apply", Write), ("bootstrap dotfiles diff", Read), ("bootstrap dotfiles edit", Write), + ("bootstrap dotfiles exclude", Write), ("bootstrap dotfiles history", Read), ("bootstrap dotfiles history describe", Write), ("bootstrap dotfiles history diff", Read), ("bootstrap dotfiles history ls", Read), ("bootstrap dotfiles history show", Read), + ("bootstrap dotfiles include", Write), ("bootstrap dotfiles paths", Read), ("bootstrap dotfiles rollback", Destructive), ("bootstrap dotfiles save", Write), @@ -95,6 +97,7 @@ pub(super) const EFFECTS: &[(&str, SpecCommandEffect)] = &[ ("bootstrap dotfiles unapply", Destructive), ("bootstrap dotfiles undo", Destructive), ("bootstrap dotfiles untrack", Write), + ("bootstrap dotfiles watch", Write), ("bootstrap linux", Read), ("bootstrap linux systemd-units", Read), ("bootstrap linux systemd-units apply", Write), diff --git a/src/cli/doctor/mod.rs b/src/cli/doctor/mod.rs index ae90b24bc67..1ccdc690a4c 100644 --- a/src/cli/doctor/mod.rs +++ b/src/cli/doctor/mod.rs @@ -66,6 +66,18 @@ enum SystemDefaultsDiagnosis { } /// outcome of the `[bootstrap.user].login_shell` doctor check +#[derive(serde::Serialize)] +struct DotfilesDiagnosis { + tracked: usize, + watcher: String, + stale: bool, + health_age_secs: Option, + unavailable: Option, + last_error: Option, + degraded: Vec, + throttled: Vec, +} + enum SystemLoginShellDiagnosis { Unavailable { reason: String, @@ -236,6 +248,9 @@ impl Doctor { if let Some(system_defaults) = self.system_defaults_json(&config).await { data.insert("system_defaults".into(), system_defaults); } + if let Some(dotfiles) = self.dotfiles_json(&config).await { + data.insert("dotfiles".into(), dotfiles); + } if let Some(system_login_shell) = self.system_login_shell_json(&config).await { data.insert("system_login_shell".into(), system_login_shell); @@ -495,6 +510,7 @@ impl Doctor { self.analyze_system_packages(config).await?; self.analyze_system_defaults(config).await?; + self.analyze_dotfiles(config).await?; self.analyze_system_login_shell(config).await?; Ok(()) @@ -649,6 +665,123 @@ impl Doctor { Ok(()) } + /// Dotfiles history health: the watcher, capture failures, and throttled + /// files, from the health the watcher persists and the store itself. + /// Inspects only: never syncs, applies, or prompts. Returns `None` when + /// nothing is tracked and no watcher is declared. + async fn check_dotfiles(&mut self, config: &Arc) -> Option { + use crate::system::history::health; + use crate::system::history::tracked::{EntryKind, TrackedSet}; + if !crate::config::Settings::get().history.enabled { + return None; + } + let tracked = TrackedSet::from_config(config).ok()?; + let watcher = crate::cli::dotfiles::capture_health::watcher().await.ok()?; + let tracks = tracked + .entries + .iter() + .filter(|entry| entry.kind == EntryKind::Track) + .count(); + if tracks == 0 && watcher == crate::cli::dotfiles::capture_health::Watcher::NotDeclared { + return None; + } + let state_dir = crate::system::history::store::state_dir(); + let unavailable = crate::system::history::checkpoint::Store::open() + .map(|store| store.unavailable().map(str::to_string)) + .unwrap_or_else(|err| Some(format!("{err:#}"))); + let health = health::read(&state_dir); + let running = watcher == crate::cli::dotfiles::capture_health::Watcher::Running; + let age = health.as_ref().and_then(health::age_secs); + let reconcile = crate::duration::parse_duration( + &crate::config::Settings::get().history.watch.reconcile, + ) + .map(|d| d.as_secs()) + .unwrap_or(600); + let stale = running && age.is_some_and(|age| reconcile > 0 && age > reconcile * 2); + let mut diagnosis = DotfilesDiagnosis { + tracked: tracks, + watcher: watcher.as_str().to_string(), + stale, + health_age_secs: age, + unavailable: unavailable.clone(), + last_error: None, + degraded: vec![], + throttled: vec![], + }; + if let Some(reason) = unavailable { + self.errors.push(format!( + "dotfiles: checkpoints cannot be saved ({reason}).\n Edits are not being protected.\n Inspect with: mise bootstrap dotfiles status" + )); + } + if watcher == crate::cli::dotfiles::capture_health::Watcher::DeclaredNotRunning { + self.warnings.push( + "dotfiles: the history watcher is declared but not running.\n Edits are not saved automatically until it runs; explicit saves still work.\n Run: mise bootstrap services apply" + .to_string(), + ); + } + if let Some(health) = &health { + let w = &health.watcher; + if let Some(error) = &w.last_error + && w.consecutive_failures > 0 + { + diagnosis.last_error = Some(error.clone()); + self.errors.push(format!( + "dotfiles: the watcher could not save a checkpoint ({error}; {} consecutive failure(s), last at {}).\n Edits since then are not protected.\n Inspect with: mise bootstrap dotfiles status", + w.consecutive_failures, + w.last_error_at.as_deref().unwrap_or("unknown") + )); + } + for degraded in &w.degraded { + diagnosis.degraded.push(degraded.clone()); + self.warnings.push(format!( + "dotfiles: {degraded}.\n Changes there are saved by reconciliation only.\n Inspect with: mise bootstrap dotfiles status" + )); + } + diagnosis.throttled = health.throttled.clone(); + } + Some(diagnosis) + } + + async fn dotfiles_json(&mut self, config: &Arc) -> Option { + let diagnosis = self.check_dotfiles(config).await?; + serde_json::to_value(diagnosis).ok() + } + + async fn analyze_dotfiles(&mut self, config: &Arc) -> eyre::Result<()> { + let Some(diagnosis) = self.check_dotfiles(config).await else { + return Ok(()); + }; + let mut lines = vec![format!( + "{} tracked entr{}, watcher {}", + diagnosis.tracked, + if diagnosis.tracked == 1 { "y" } else { "ies" }, + diagnosis.watcher + )]; + if diagnosis.stale { + lines.push(format!( + "health information is stale (last update {} ago); the watcher may be stuck", + crate::system::history::watch::runtime::humantime(std::time::Duration::from_secs( + diagnosis.health_age_secs.unwrap_or(0) + )) + )); + } + // informational: throttling protects the history, it does not + // compromise it + for throttled in &diagnosis.throttled { + lines.push(format!( + "{} changes constantly: saved every {} ({} unsaved change(s); last saved {}). Not a failure; `mise bootstrap dotfiles exclude` if it is a log, cache, or database", + throttled.path, + crate::system::history::watch::runtime::humantime(std::time::Duration::from_secs( + throttled.interval_secs + )), + throttled.pending_changes, + throttled.last_saved.as_deref().unwrap_or("never") + )); + } + info::section("dotfiles", lines.join("\n"))?; + Ok(()) + } + /// Shared `[bootstrap.user].login_shell` check for the text and JSON doctor paths. /// Pushes the relevant warnings; returns None when nothing is configured. async fn check_system_login_shell( diff --git a/src/cli/dotfiles/capture_health.rs b/src/cli/dotfiles/capture_health.rs index abdf3bea548..1f9d3428e4d 100644 --- a/src/cli/dotfiles/capture_health.rs +++ b/src/cli/dotfiles/capture_health.rs @@ -1,11 +1,17 @@ //! Whether edits are being saved automatically, and what to do when not. +use eyre::Result; use serde::Serialize; +use crate::config::Config; + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "kebab-case")] pub(crate) enum Watcher { + /// A watcher holds the store's watch lock. Running, + /// `[bootstrap.services]` declares the built-in watcher, but none runs. + DeclaredNotRunning, NotDeclared, } @@ -13,31 +19,54 @@ impl Watcher { pub(crate) fn as_str(self) -> &'static str { match self { Self::Running => "running", + Self::DeclaredNotRunning => "declared but not running", Self::NotDeclared => "not declared", } } } -/// The watcher's state. No watcher service exists in this version, so -/// automatic capture is never running. -pub(crate) fn watcher() -> Watcher { - Watcher::NotDeclared +/// The watcher's state for the current store and configuration. +pub(crate) async fn watcher() -> Result { + let running = crate::system::history::watch::runtime::is_running( + &crate::system::history::store::state_dir(), + ); + if running { + return Ok(Watcher::Running); + } + let config = Config::get().await?; + let declared = crate::system::services_common::compose_user_declarations(&config)? + .values() + .any(|(declaration, _)| { + declaration.builtin.as_deref() == Some("history-watch") + && declaration.state != crate::system::services_common::ServiceState::Absent + }); + Ok(if declared { + Watcher::DeclaredNotRunning + } else { + Watcher::NotDeclared + }) } /// The next step for each state. pub(crate) fn advice(state: Watcher) -> &'static str { match state { Watcher::Running => "edits are saved automatically", + Watcher::DeclaredNotRunning => { + "the history watcher is declared but not running: run `mise bootstrap services apply`" + } Watcher::NotDeclared => { - "automatic capture is inactive: edits are saved by `mise bootstrap dotfiles save` and by every mutating bootstrap command" + "automatic capture is inactive: declare `[bootstrap.services.mise-history] builtin = \"history-watch\"` and run `mise bootstrap`; until then edits are saved by `mise bootstrap dotfiles save` or `mise bootstrap dotfiles watch --once`" } } } /// Warns when enrollment succeeded but nothing saves edits automatically. -pub(crate) fn report() { - let state = watcher(); - if state != Watcher::Running { - warn!("history: {}", advice(state)); +pub(crate) async fn report() { + match watcher().await { + Ok(Watcher::Running) => {} + Ok(state) => warn!("history: {}", advice(state)), + Err(err) => { + warn!("history: could not determine whether edits are saved automatically: {err:#}") + } } } diff --git a/src/cli/dotfiles/exclude.rs b/src/cli/dotfiles/exclude.rs new file mode 100644 index 00000000000..9290dfac95a --- /dev/null +++ b/src/cli/dotfiles/exclude.rs @@ -0,0 +1,36 @@ +use eyre::Result; + +/// Never capture paths matching a glob +/// +/// Adds the glob to `[history] exclude` in the global config. Use it for +/// logs, caches, databases, and constantly rewritten application state; a +/// file that genuinely holds configuration but changes constantly is +/// better tracked with `--no-autosave` and saved explicitly. +#[derive(Debug, usage_rs::Args)] +#[usage(verbatim_doc_comment)] +pub(crate) struct DotfilesExclude { + /// A glob such as `~/.config/hypr/plugins/**` + glob: String, +} + +impl DotfilesExclude { + pub(crate) async fn run(self) -> Result<()> { + super::paths::edit_exclude(&self.glob, true) + } +} + +/// Capture paths matching a glob again +/// +/// Removes the glob from `[history] exclude` in the global config. +#[derive(Debug, usage_rs::Args)] +#[usage(verbatim_doc_comment)] +pub(crate) struct DotfilesInclude { + /// The glob as written by `exclude` + glob: String, +} + +impl DotfilesInclude { + pub(crate) async fn run(self) -> Result<()> { + super::paths::edit_exclude(&self.glob, false) + } +} diff --git a/src/cli/dotfiles/history_status.rs b/src/cli/dotfiles/history_status.rs index f5755e42a98..ba9be118760 100644 --- a/src/cli/dotfiles/history_status.rs +++ b/src/cli/dotfiles/history_status.rs @@ -17,6 +17,8 @@ pub(crate) struct HistoryReport { pub pending_operations: usize, pub watcher: super::capture_health::Watcher, pub unavailable: Option, + /// What the watcher last persisted, if anything. + pub health: Option, } #[derive(Serialize)] @@ -49,8 +51,9 @@ pub(crate) async fn report() -> Result { checkpoints: entries.len(), latest, pending_operations, - watcher: super::capture_health::watcher(), + watcher: super::capture_health::watcher().await?, unavailable: store.unavailable().map(str::to_string), + health: crate::system::history::health::read(store.state_dir()), }) } @@ -91,5 +94,56 @@ pub(crate) fn print(report: &HistoryReport) -> Result<()> { report.watcher.as_str(), super::capture_health::advice(report.watcher) ); + if let Some(health) = &report.health { + use crate::system::history::health::age_secs; + use crate::system::history::watch::runtime::humantime; + let age = age_secs(health) + .map(|secs| format!("{} ago", humantime(std::time::Duration::from_secs(secs)))) + .unwrap_or_else(|| "at an unknown time".into()); + let w = &health.watcher; + miseprintln!( + " watcher health (updated {age}): last capture {}, last reconcile {}{}", + w.last_capture + .as_deref() + .map(local_time) + .unwrap_or_else(|| "never".into()), + w.last_reconcile + .as_deref() + .map(local_time) + .unwrap_or_else(|| "never".into()), + if report.watcher == super::capture_health::Watcher::Running { + "" + } else { + "; the watcher is not running now, so this is what it last reported" + } + ); + if let Some(error) = &w.last_error { + miseprintln!( + " last capture failure: {error} ({} consecutive; at {}). Edits since then are not protected.", + w.consecutive_failures, + w.last_error_at + .as_deref() + .map(local_time) + .unwrap_or_else(|| "unknown".into()) + ); + } + for degraded in &w.degraded { + miseprintln!(" degraded: {degraded}"); + } + for throttled in &health.throttled { + miseprintln!( + " throttled: {} changes constantly; saved every {} ({} unsaved change(s), last saved {}). Not a failure: `mise bootstrap dotfiles exclude '{}'` if it is a log, cache, or database, or track it with --no-autosave and save explicitly.", + throttled.path, + humantime(std::time::Duration::from_secs(throttled.interval_secs)), + throttled.pending_changes, + throttled + .last_saved + .as_deref() + .map(local_time) + .unwrap_or_else(|| "never".into()), + throttled.path + ); + } + } Ok(()) } diff --git a/src/cli/dotfiles/mod.rs b/src/cli/dotfiles/mod.rs index b4b7ddd2235..811b0dd98dc 100644 --- a/src/cli/dotfiles/mod.rs +++ b/src/cli/dotfiles/mod.rs @@ -6,6 +6,7 @@ mod apply; pub(crate) mod capture_health; mod diff; mod edit; +mod exclude; pub(crate) mod history; mod history_status; mod paths; @@ -16,11 +17,13 @@ pub(crate) mod track; mod unapply; mod undo; mod untrack; +mod watch; pub(crate) use add::DotfilesAdd; pub(crate) use apply::DotfilesApply; pub(crate) use diff::DotfilesDiff; pub(crate) use edit::DotfilesEdit; +pub(crate) use exclude::{DotfilesExclude, DotfilesInclude}; pub(crate) use history::DotfilesHistory; pub(crate) use paths::DotfilesPaths; pub(crate) use rollback::DotfilesRollback; @@ -30,6 +33,7 @@ pub(crate) use track::DotfilesTrack; pub(crate) use unapply::DotfilesUnapply; pub(crate) use undo::DotfilesUndo; pub(crate) use untrack::DotfilesUntrack; +pub(crate) use watch::DotfilesWatch; /// Load, validate, and filter whole-file and edit requests with the same /// target semantics for every command that acts on both kinds of entry. diff --git a/src/cli/dotfiles/paths.rs b/src/cli/dotfiles/paths.rs index 9c8869f037e..57484e9e332 100644 --- a/src/cli/dotfiles/paths.rs +++ b/src/cli/dotfiles/paths.rs @@ -24,6 +24,10 @@ pub(crate) struct DotfilesPaths { /// Show what tracking this path would capture #[usage(long, value_name = "PATH")] preview: Option, + + /// List the paths the watcher found changing constantly + #[usage(long)] + noisy: bool, } #[derive(Serialize)] @@ -46,6 +50,9 @@ struct PathRow { impl DotfilesPaths { pub(crate) async fn run(self) -> Result<()> { + if self.noisy { + return self.print_noisy(); + } let tracked = match &self.preview { Some(path) => { let mut set = TrackedSet { @@ -167,3 +174,61 @@ impl DotfilesPaths { Ok(()) } } + +impl DotfilesPaths { + fn print_noisy(&self) -> Result<()> { + use crate::system::history::watch::{noise, runtime}; + let record = noise::read(&runtime::noisy_path_in( + &crate::system::history::store::state_dir(), + )); + if self.json { + miseprintln!("{}", serde_json::to_string_pretty(&record.paths)?); + return Ok(()); + } + if record.paths.is_empty() { + info!("the watcher is not throttling any path"); + return Ok(()); + } + let mut table = MiseTable::new( + false, + &["Path", "Saving every", "Unsaved changes", "Last seen"], + ); + for (path, noisy) in &record.paths { + table.add_row(vec![ + path.clone(), + crate::system::history::watch::runtime::humantime(std::time::Duration::from_secs( + noisy.interval_secs, + )), + noisy.pending_changes.to_string(), + noisy.last_seen.clone(), + ]); + } + table.print()?; + miseprintln!( + "These paths keep changing, so the watcher saves them ever more rarely (never excluded or switched to manual saving on its own). Exclude a log, cache, or database with `mise bootstrap dotfiles exclude ''`; track configuration that changes constantly with `--no-autosave` and save it explicitly." + ); + Ok(()) + } +} + +pub(crate) fn edit_exclude(glob: &str, add: bool) -> Result<()> { + let glob = glob.trim(); + if glob.is_empty() { + eyre::bail!("a glob is required"); + } + let global = crate::config::global_config_path(); + let changed = crate::cli::dotfiles::track::edit_exclude(glob, add)?; + match (add, changed) { + (true, true) => info!( + "history: {glob} is excluded from capture ({})", + display_path(&global) + ), + (true, false) => info!("history: {glob} was already excluded"), + (false, true) => info!( + "history: {glob} is captured again ({})", + display_path(&global) + ), + (false, false) => info!("history: {glob} was not excluded"), + } + Ok(()) +} diff --git a/src/cli/dotfiles/track.rs b/src/cli/dotfiles/track.rs index 5247c2feeaf..0652c98442e 100644 --- a/src/cli/dotfiles/track.rs +++ b/src/cli/dotfiles/track.rs @@ -142,7 +142,7 @@ impl DotfilesTrack { } return Err(err.wrap_err(format!("{} was left unchanged", display_path(&config_path)))); } - crate::cli::dotfiles::capture_health::report(); + crate::cli::dotfiles::capture_health::report().await; Ok(()) } @@ -330,3 +330,41 @@ static AFTER_LONG_HELP: &str = color_print::cstr!( $ mise bootstrap dotfiles track ~/.ssh/config --no-share "# ); + +/// Adds (or removes) a glob in `[history] exclude` of the global config. +/// Returns whether the file changed. +pub(crate) fn edit_exclude(glob: &str, add: bool) -> Result { + use toml_edit::{Item, Value}; + let global = crate::config::global_config_path(); + let mut doc = read_document(&global)?; + let history = doc + .entry("history") + .or_insert(Item::Table(toml_edit::Table::new())); + let Some(table) = history.as_table_mut() else { + eyre::bail!("[history] in {} is not a table", display_path(&global)); + }; + table.set_implicit(false); + let exclude = table + .entry("exclude") + .or_insert(Item::Value(Value::Array(toml_edit::Array::new()))); + let Some(array) = exclude.as_array_mut() else { + eyre::bail!( + "[history] exclude in {} is not an array", + display_path(&global) + ); + }; + let present = array.iter().any(|value| value.as_str() == Some(glob)); + let changed = if add && !present { + array.push(Value::String(toml_edit::Formatted::new(glob.to_string()))); + true + } else if !add && present { + array.retain(|value| value.as_str() != Some(glob)); + true + } else { + false + }; + if changed { + crate::file::write(&global, doc.to_string())?; + } + Ok(changed) +} diff --git a/src/cli/dotfiles/untrack.rs b/src/cli/dotfiles/untrack.rs index 1d5ae95e01b..ec8fd42a997 100644 --- a/src/cli/dotfiles/untrack.rs +++ b/src/cli/dotfiles/untrack.rs @@ -103,24 +103,7 @@ impl DotfilesUntrack { } else { key.clone() }; - let mut doc = super::track::read_document(&global)?; - let history = doc - .entry("history") - .or_insert(Item::Table(toml_edit::Table::new())); - if let Some(table) = history.as_table_mut() { - table.set_implicit(false); - let exclude = table - .entry("exclude") - .or_insert(Item::Value(Value::Array(toml_edit::Array::new()))); - if let Some(array) = exclude.as_array_mut() - && !array - .iter() - .any(|value| value.as_str() == Some(glob.as_str())) - { - array.push(Value::String(toml_edit::Formatted::new(glob.clone()))); - } - } - file::write(&global, doc.to_string())?; + super::track::edit_exclude(&glob, true)?; info!( "dotfiles: {key} is covered by {} ({}); excluded it in {}", owner.display(), diff --git a/src/cli/dotfiles/watch.rs b/src/cli/dotfiles/watch.rs new file mode 100644 index 00000000000..092740da193 --- /dev/null +++ b/src/cli/dotfiles/watch.rs @@ -0,0 +1,58 @@ +use eyre::Result; + +use crate::system::history::watch::runtime::{self, WatchOptions}; + +/// Save tracked files as they change +/// +/// Runs in the foreground: installs filesystem watches for every autosaved +/// tracked entry, saves a checkpoint once a changed file has been quiet for +/// `history.watch.debounce` (a file that keeps changing never delays the +/// others; `history.watch.max_interval` saves it regardless), and +/// reconciles the whole set at startup, every `history.watch.reconcile`, +/// and when the configuration changes. Manual-save entries are never +/// watched. +/// +/// The `history-watch` built-in service runs this for you: +/// +/// [bootstrap.services.mise-history] +/// builtin = "history-watch" +/// +/// Exit codes: 0 when history is disabled or another watcher already runs; +/// 1 when git is unusable, the store cannot open, or no watch can be +/// installed. A capture that fails is retried with backoff and never drops +/// the pending changes; one that would overlap another history operation +/// is deferred. +#[derive(Debug, usage_rs::Args)] +#[usage(verbatim_doc_comment, after_long_help = AFTER_LONG_HELP)] +pub(crate) struct DotfilesWatch { + /// Reconcile once and exit (for timers and cron) + #[usage(long)] + once: bool, + + /// One JSON object per line instead of log lines + #[usage(long, short = 'J')] + json: bool, +} + +impl DotfilesWatch { + pub(crate) async fn run(self) -> Result<()> { + let code = runtime::run(WatchOptions { + once: self.once, + json: self.json, + }) + .await?; + if code != 0 { + return Err(crate::request_exit(code)); + } + Ok(()) + } +} + +static AFTER_LONG_HELP: &str = color_print::cstr!( + r#"Examples: + + $ mise bootstrap dotfiles watch + $ mise bootstrap dotfiles watch --once # one reconcile, for a timer + $ mise bootstrap dotfiles watch --json +"# +); diff --git a/src/system/history/checkpoint.rs b/src/system/history/checkpoint.rs index dcaf98a1229..46d3711fa01 100644 --- a/src/system/history/checkpoint.rs +++ b/src/system/history/checkpoint.rs @@ -53,6 +53,11 @@ pub(crate) struct Draft { /// Paths named explicitly: manual-save entries covering them are read /// live and promoted, becoming their new saved version. pub explicit_paths: Vec, + /// Paths the watcher is throttling: their files are carried forward + /// from the newest checkpoint instead of read live, so a capture for + /// another path never defeats their schedule. Ignored by protective + /// captures and explicit saves. + pub held: Vec, } impl Draft { @@ -256,6 +261,19 @@ impl Store { modes.entry(path).or_insert(bits); } } + // a held path keeps the mode the previous checkpoint recorded (or + // none), like its content + if !draft.held.is_empty() && !draft.protective && draft.explicit_paths.is_empty() { + for held in &draft.held { + let display = display_path(held); + modes.remove(&display); + if let Some((previous_checkpoint, _)) = &previous_tree + && let Some(bits) = previous_checkpoint.tree.modes.get(&display) + { + modes.insert(display, *bits); + } + } + } let mut coverage = tracked.coverage(&walk); let promoted_head = match &self.repo { Some(repo) => repo.promoted_head()?, @@ -295,6 +313,12 @@ impl Store { } } } + let composed = self.hold_paths( + repo, + &composed, + &draft, + previous_tree.as_ref().map(|(_, tree)| tree.as_str()), + )?; (Some(composed), result.roots, true, None) } Err(err) => { @@ -462,6 +486,39 @@ impl Store { /// versions, and promotes the ones the draft names explicitly. A /// promotion is durable (a new commit on `refs/promoted`) before the /// checkpoint referencing it is written. + /// Carries the draft's held paths forward from the previous checkpoint: + /// their live content is not what this capture records. Protective + /// captures and explicit saves hold nothing. + /// Held paths (files whose save is not due yet) take their previous + /// version, or previous absence, from the newest checkpoint: a change, + /// a creation, and a deletion are all held until the path is due. Held + /// paths are files, matched exactly. + fn hold_paths( + &self, + repo: &HistoryRepo, + tree: &str, + draft: &Draft, + previous: Option<&str>, + ) -> Result { + if draft.held.is_empty() || draft.protective || !draft.explicit_paths.is_empty() { + return Ok(tree.to_string()); + } + // with no previous checkpoint there is nothing to carry forward: + // the live content is the only content there is + let Some(previous) = previous else { + return Ok(tree.to_string()); + }; + let mut overlays = vec![]; + for held in &draft.held { + let tree_path = display_to_tree_path(&display_path(held)); + overlays.push(Overlay { + object: repo.object_at(previous, &tree_path)?, + path: tree_path, + }); + } + repo.compose(tree, &overlays) + } + fn compose_manual( &self, repo: &HistoryRepo, diff --git a/src/system/history/health.rs b/src/system/history/health.rs new file mode 100644 index 00000000000..ab765b7e886 --- /dev/null +++ b/src/system/history/health.rs @@ -0,0 +1,81 @@ +//! Background health, persisted under `$MISE_STATE_DIR/history/health.json` +//! by the watcher and read by `mise doctor` and `mise bootstrap dotfiles +//! status`. This is pull-based visibility: nothing here gets the user's +//! attention on its own. Readers distinguish stale information (the last +//! update is older than the watcher's reconcile period while a watcher +//! still holds the lock) from confirmed current health. + +use std::path::{Path, PathBuf}; + +use eyre::Result; +use serde::{Deserialize, Serialize}; + +use super::store; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct Health { + /// When this record was written (RFC 3339). + #[serde(default)] + pub updated_at: String, + #[serde(default)] + pub watcher: WatcherHealth, + /// Paths whose autosave interval is stretched by sustained churn. + #[serde(default)] + pub throttled: Vec, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct WatcherHealth { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_capture: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_reconcile: Option, + /// The last capture failure, with when it happened; cleared by the + /// next successful capture. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error_at: Option, + /// Consecutive capture failures. + #[serde(default)] + pub consecutive_failures: u32, + #[serde(default)] + pub degraded: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct ThrottledPath { + pub path: String, + pub interval_secs: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_saved: Option, + /// Changes seen since the last save. + /// Changes seen since the last save (events, not verified content + /// differences). + pub pending_changes: u32, + /// The interval reached the heavy-throttling mark. + pub heavy: bool, +} + +pub(crate) fn path_in(state_dir: &Path) -> PathBuf { + store::store_dir_in(state_dir).join("health.json") +} + +pub(crate) fn read(state_dir: &Path) -> Option { + let text = std::fs::read_to_string(path_in(state_dir)).ok()?; + serde_json::from_str(&text).ok() +} + +pub(crate) fn write(state_dir: &Path, health: &mut Health) -> Result<()> { + health.updated_at = store::now_rfc3339(); + store::write_json(&path_in(state_dir), health) +} + +/// How old a record is, in seconds, if its timestamp parses. +pub(crate) fn age_secs(health: &Health) -> Option { + let updated = chrono::DateTime::parse_from_rfc3339(&health.updated_at).ok()?; + let age = chrono::Utc::now().signed_duration_since(updated); + u64::try_from(age.num_seconds()).ok() +} diff --git a/src/system/history/mod.rs b/src/system/history/mod.rs index 767e9459d0b..752b48b4ad1 100644 --- a/src/system/history/mod.rs +++ b/src/system/history/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod checkpoint; pub(crate) mod config; +pub(crate) mod health; pub(crate) mod journal; pub(crate) mod replay; pub(crate) mod retention; @@ -19,5 +20,6 @@ pub(crate) mod select; pub(crate) mod shadow; pub(crate) mod store; pub(crate) mod tracked; +pub(crate) mod watch; pub(crate) use scope::OperationScope; diff --git a/src/system/history/store.rs b/src/system/history/store.rs index 27ac25d0ae4..27154bc2684 100644 --- a/src/system/history/store.rs +++ b/src/system/history/store.rs @@ -18,6 +18,11 @@ use crate::file::{self, display_path}; pub(crate) const SCHEMA_VERSION: u32 = 1; +/// The state directory the store lives under. +pub(crate) fn state_dir() -> PathBuf { + crate::dirs::STATE.to_path_buf() +} + pub(crate) fn store_dir_in(state_dir: &Path) -> PathBuf { state_dir.join("history") } diff --git a/src/system/history/tracked.rs b/src/system/history/tracked.rs index ccd7dac2203..028777e863a 100644 --- a/src/system/history/tracked.rs +++ b/src/system/history/tracked.rs @@ -86,7 +86,7 @@ impl TrackedEntry { display_path(&self.path) } - fn new(path: PathBuf, kind: EntryKind, mode: &str, policy: Policy) -> Self { + pub(crate) fn new(path: PathBuf, kind: EntryKind, mode: &str, policy: Policy) -> Self { Self { path, kind, @@ -857,6 +857,29 @@ pub(crate) fn normalize_target(path: &Path) -> PathBuf { dunce::canonicalize(&expanded).unwrap_or_else(|_| lexical(&expanded)) } +/// Where a chain of symlinks ends, lexically: the last link's target, +/// which need not exist (a target between two versions, say). +pub(crate) fn link_target(link: &Path) -> Option { + let mut current = link.to_path_buf(); + let mut seen = BTreeSet::new(); + for _ in 0..MAX_LINK_DEPTH { + if !seen.insert(current.clone()) { + return None; + } + let dest = std::fs::read_link(¤t).ok()?; + let joined = if dest.is_absolute() { + dest + } else { + current.parent()?.join(dest) + }; + current = lexical(&joined); + if !current.is_symlink() { + return Some(current); + } + } + None +} + fn lexical(path: &Path) -> PathBuf { let mut out = PathBuf::new(); for component in path.components() { diff --git a/src/system/history/watch/mod.rs b/src/system/history/watch/mod.rs new file mode 100644 index 00000000000..18faeadf3b6 --- /dev/null +++ b/src/system/history/watch/mod.rs @@ -0,0 +1,8 @@ +//! The history watcher: filesystem watches for the tracked set (`plan`), +//! per-file adaptive save scheduling (`schedule`), the report of throttled +//! paths (`noise`), and the foreground process (`runtime`). + +pub(crate) mod noise; +pub(crate) mod plan; +pub(crate) mod runtime; +pub(crate) mod schedule; diff --git a/src/system/history/watch/noise.rs b/src/system/history/watch/noise.rs new file mode 100644 index 00000000000..64b58395e1e --- /dev/null +++ b/src/system/history/watch/noise.rs @@ -0,0 +1,39 @@ +//! The persisted report of throttled paths, written by the watcher and +//! listed by `mise bootstrap dotfiles paths --noisy`. A noisy path is never +//! excluded automatically and never switched to manual-save; the report +//! says what the watcher stretched and lets the user decide. + +use std::collections::BTreeMap; +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct NoisyRecord { + #[serde(default)] + pub paths: BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct NoisyPath { + /// The stretched autosave interval. + pub interval_secs: u64, + /// Changes seen since the last save. + #[serde(default)] + pub pending_changes: u32, + pub last_seen: String, +} + +pub(crate) fn read(path: &Path) -> NoisyRecord { + std::fs::read_to_string(path) + .ok() + .and_then(|text| serde_json::from_str(&text).ok()) + .unwrap_or_default() +} + +pub(crate) fn write(path: &Path, record: &NoisyRecord) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, serde_json::to_string_pretty(record)?) +} diff --git a/src/system/history/watch/plan.rs b/src/system/history/watch/plan.rs new file mode 100644 index 00000000000..8675511ac2c --- /dev/null +++ b/src/system/history/watch/plan.rs @@ -0,0 +1,205 @@ +//! Which directories the watcher installs watches on, computed from the +//! tracked set without touching the watcher: a pure function of the +//! entries and what exists on disk. + +use std::path::{Path, PathBuf}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Mode { + /// The directory and everything below it. + Recursive, + /// The directory's own entries only (a tracked file's parent, or the + /// nearest existing ancestor of a path that does not exist yet). + Flat, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Anchor { + pub path: PathBuf, + pub mode: Mode, +} + +/// What a tracked path is right now. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PathKind { + Directory, + File, + Missing, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct WatchPlan { + pub anchors: Vec, + /// Tracked paths that do not exist yet; their creation is seen through + /// a flat anchor on the nearest existing ancestor, after which the plan + /// is rebuilt. + pub pending: Vec, +} + +impl WatchPlan { + /// `nearest_existing` answers, for a missing path, which ancestor + /// exists (injected so the plan stays testable without a filesystem). + pub(crate) fn build( + paths: impl IntoIterator, + nearest_existing: impl Fn(&Path) -> Option, + ) -> Self { + let mut anchors: Vec = vec![]; + let mut pending = vec![]; + for (path, kind) in paths { + match kind { + PathKind::Directory => anchors.push(Anchor { + path, + mode: Mode::Recursive, + }), + PathKind::File => { + if let Some(parent) = path.parent() { + anchors.push(Anchor { + path: parent.to_path_buf(), + mode: Mode::Flat, + }); + } + } + PathKind::Missing => { + if let Some(ancestor) = nearest_existing(&path) { + anchors.push(Anchor { + path: ancestor, + mode: Mode::Flat, + }); + } + pending.push(path); + } + } + } + anchors.sort_by(|a, b| a.path.cmp(&b.path).then_with(|| a.mode.cmp(&b.mode))); + // a recursive anchor covers every anchor below it; on the same path + // recursive wins over flat + let mut merged: Vec = vec![]; + for anchor in anchors { + if merged.iter().any(|kept| { + kept.mode == Mode::Recursive + && (anchor.path == kept.path || anchor.path.starts_with(&kept.path)) + }) { + continue; + } + if let Some(existing) = merged.iter_mut().find(|kept| kept.path == anchor.path) { + if anchor.mode == Mode::Recursive { + existing.mode = Mode::Recursive; + // anchors below a newly recursive one are now covered + let path = existing.path.clone(); + merged.retain(|kept| kept.path == path || !kept.path.starts_with(&path)); + } + continue; + } + merged.push(anchor); + } + pending.sort(); + pending.dedup(); + Self { + anchors: merged, + pending, + } + } +} + +impl PartialOrd for Mode { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Mode { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // recursive sorts first so it is kept when paths tie + let rank = |mode: &Mode| match mode { + Mode::Recursive => 0, + Mode::Flat => 1, + }; + rank(self).cmp(&rank(other)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(s: &str) -> PathBuf { + PathBuf::from(s) + } + + #[test] + fn directories_are_recursive_and_files_watch_their_parent() { + let plan = WatchPlan::build( + [ + (p("/home/u/.config/hypr"), PathKind::Directory), + (p("/home/u/.zshrc"), PathKind::File), + (p("/home/u/.gitconfig"), PathKind::File), + ], + |_| None, + ); + assert_eq!( + plan.anchors, + vec![ + Anchor { + path: p("/home/u"), + mode: Mode::Flat + }, + Anchor { + path: p("/home/u/.config/hypr"), + mode: Mode::Recursive + }, + ] + ); + assert!(plan.pending.is_empty()); + } + + #[test] + fn a_recursive_anchor_absorbs_anchors_below_it() { + let plan = WatchPlan::build( + [ + (p("/home/u/.config/hypr/bindings.lua"), PathKind::File), + (p("/home/u/.config"), PathKind::Directory), + (p("/home/u/.config/mise"), PathKind::Directory), + ], + |_| None, + ); + assert_eq!( + plan.anchors, + vec![Anchor { + path: p("/home/u/.config"), + mode: Mode::Recursive + }] + ); + } + + #[test] + fn missing_paths_are_pending_and_watched_through_an_ancestor() { + let plan = WatchPlan::build( + [(p("/home/u/.config/later/file"), PathKind::Missing)], + |path| { + assert_eq!(path, Path::new("/home/u/.config/later/file")); + Some(p("/home/u/.config")) + }, + ); + assert_eq!( + plan.anchors, + vec![Anchor { + path: p("/home/u/.config"), + mode: Mode::Flat + }] + ); + assert_eq!(plan.pending, vec![p("/home/u/.config/later/file")]); + } + + #[test] + fn recursive_wins_over_flat_on_the_same_path() { + let plan = WatchPlan::build( + [ + (p("/home/u/.config/a"), PathKind::File), + (p("/home/u/.config"), PathKind::Directory), + ], + |_| None, + ); + assert_eq!(plan.anchors.len(), 1); + assert_eq!(plan.anchors[0].mode, Mode::Recursive); + } +} diff --git a/src/system/history/watch/runtime.rs b/src/system/history/watch/runtime.rs new file mode 100644 index 00000000000..5f72b744156 --- /dev/null +++ b/src/system/history/watch/runtime.rs @@ -0,0 +1,1230 @@ +//! The foreground watcher process behind `mise bootstrap dotfiles watch` and +//! the `history-watch` built-in service: installs filesystem watches for the +//! tracked set, schedules each changed path on its own (a constantly +//! rewritten file is saved ever more rarely, never floods the history, and +//! never delays an ordinary edit), saves checkpoints, and persists its +//! health for `mise doctor` and `mise bootstrap dotfiles status`. Captures +//! never wait on the network and never run while another history operation +//! holds the operation lock; they are deferred and retried. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use eyre::{Result, bail}; +use notify::{RecommendedWatcher, RecursiveMode}; +use notify_debouncer_full::{DebounceEventResult, Debouncer, NoCache, new_debouncer_opt}; +use serde_json::json; +use tokio::sync::mpsc; + +use super::noise::{self, NoisyPath, NoisyRecord}; +use super::plan::{Anchor, Mode, PathKind, WatchPlan}; +use super::schedule::{self, Adjustment, Limits, PersistedSchedule, Schedule}; +use crate::config::{Config, Settings}; +use crate::file::display_path; +use crate::lock_file::LockFile; +use crate::system::history::checkpoint::{Draft, Outcome, Store}; +use crate::system::history::health::{self, Health, ThrottledPath}; +use crate::system::history::store::{self, Trigger}; +use crate::system::history::tracked::{ + self, ExcludeSet, TrackedSet, hard_exclusions, normalize, normalize_target, +}; + +/// How long the debouncer coalesces raw filesystem events before they reach +/// the scheduler, which applies the configured quiet period on top. +const COALESCE: Duration = Duration::from_millis(500); +const BACKOFF_MIN: Duration = Duration::from_secs(1); +const BACKOFF_MAX: Duration = Duration::from_secs(5 * 60); +/// How often a start retries the watch lock a status probe may be holding +/// for a moment. +const WATCH_LOCK_TRIES: u32 = 5; +const WATCH_LOCK_RETRY: Duration = Duration::from_millis(200); + +/// How often, and how many times, the shutdown capture waits for a running +/// history operation to finish before giving up. +const SHUTDOWN_RETRY_EVERY: Duration = Duration::from_secs(1); +const SHUTDOWN_RETRIES: usize = 10; + +/// What became of a capture attempt. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Attempt { + /// It ran (a checkpoint was written, or nothing had changed). + Done, + /// Another history operation holds the lock; retried later. + Deferred, + /// It failed; retried after the backoff. + Failed, +} + +pub(crate) struct WatchOptions { + /// Reconcile once and exit. + pub once: bool, + /// One JSON object per line instead of log lines. + pub json: bool, +} + +/// Runs the watcher; returns the process exit code. +pub(crate) async fn run(opts: WatchOptions) -> Result { + let out = Output { json: opts.json }; + if !Settings::get().history.enabled { + out.emit( + "disabled", + "history is disabled (history.enabled = false)", + json!({}), + ); + return Ok(0); + } + let store = Store::open()?; + if let Some(reason) = store.unavailable() { + out.emit("unavailable", &format!("cannot watch: {reason}"), json!({})); + return Ok(1); + } + // a status probe (`doctor`, `status`, `track`) takes the lock for a + // moment to see whether a watcher holds it: a start that lands on that + // moment tries again rather than concluding another watcher runs + let mut watch_lock = None; + for attempt in 0..WATCH_LOCK_TRIES { + if let Some(lock) = LockFile::new(&watch_lock_in(store.state_dir())).try_lock()? { + watch_lock = Some(lock); + break; + } + if attempt + 1 < WATCH_LOCK_TRIES { + tokio::time::sleep(WATCH_LOCK_RETRY).await; + } + } + let Some(_watch_lock) = watch_lock else { + out.emit( + "already-running", + "another watcher is running for this store", + json!({}), + ); + return Ok(0); + }; + let settings = Settings::get(); + let mut intervals = Intervals::from_settings(&settings); + let mut state = State::load().await?; + let mut capture = Capture::new(store, out, intervals.limits.clone()); + capture.health.watcher.started_at = Some(store::now_rfc3339()); + if opts.once { + // the restored schedule applies to this capture too: a throttled + // file whose save is not due is held, not read live + let outcome = capture.reconcile(&state.tracked, "startup reconcile"); + capture.health.watcher.last_reconcile = Some(store::now_rfc3339()); + capture.write_health(); + return Ok(match outcome { + Attempt::Done => 0, + Attempt::Deferred => { + capture.out.emit( + "unsaved", + "nothing was saved: another history operation is running; run again once it finished", + json!({ "reason": "deferred" }), + ); + 1 + } + Attempt::Failed => { + capture.out.emit( + "unsaved", + &format!( + "nothing was saved: {}", + capture + .health + .watcher + .last_error + .as_deref() + .unwrap_or("the capture failed") + ), + json!({ "reason": "failed" }), + ); + 1 + } + }); + } + + let (tx, mut rx) = mpsc::unbounded_channel::(); + let mut debouncer = new_debouncer_opt::<_, RecommendedWatcher, NoCache>( + COALESCE, + None, + move |result| { + let _ = tx.send(result); + }, + NoCache, + notify::Config::default(), + )?; + let mut installed = match install(&mut debouncer, &[], &state.plan.anchors, &mut capture) { + Ok(installed) if !installed.is_empty() => installed, + outcome => { + // what changed while the watcher was down is saved before it + // gives up on watching, and why it gave up is on record for + // `doctor` and `status` + capture.reconcile(&state.tracked, "startup reconcile"); + capture.health.watcher.last_reconcile = Some(store::now_rfc3339()); + let reason = match &outcome { + Ok(_) => "no watch could be installed for the tracked set".to_string(), + Err(err) => format!("{err:#}"), + }; + capture.health.watcher.last_error = Some(reason.clone()); + capture.health.watcher.last_error_at = Some(store::now_rfc3339()); + capture.health.watcher.consecutive_failures += 1; + capture.write_health(); + outcome?; + bail!("{reason}"); + } + }; + // the first capture comes after the watches are in place, so an edit + // landing between the two reaches the scheduler instead of waiting for + // the next reconcile + capture.reconcile(&state.tracked, "startup reconcile"); + capture.health.watcher.last_reconcile = Some(store::now_rfc3339()); + capture.out.emit( + "started", + &format!( + "watching {} anchor(s) for {} tracked entr{}", + installed.len(), + state.tracked.entries.len(), + if state.tracked.entries.len() == 1 { + "y" + } else { + "ies" + } + ), + json!({ "anchors": installed.len(), "pending": state.plan.pending.len() }), + ); + capture.write_health(); + + let mut shutdown = Shutdown::new()?; + let mut next_reconcile = intervals + .reconcile + .map(|every| tokio::time::Instant::now() + every); + loop { + // the next save, or the retry of a deferred or failed capture, + // whichever comes first + let flush_at = match ( + capture.schedule.deadline().map(|at| capture.not_before(at)), + capture.retry_due(), + ) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }; + let flush = async { + match flush_at { + Some(at) => tokio::time::sleep_until(tokio::time::Instant::from_std(at)).await, + None => std::future::pending::<()>().await, + } + }; + let reconcile = async { + match next_reconcile { + Some(at) => tokio::time::sleep_until(at).await, + None => std::future::pending::<()>().await, + } + }; + tokio::select! { + received = rx.recv() => { + let Some(result) = received else { + capture.out.emit( + "error", + "the filesystem watch stopped delivering events; stopping so the service restarts it", + json!({ "message": "watch channel closed" }), + ); + finish(&mut capture, &state.tracked).await; + // recorded after the final capture, which would clear it + capture.health.watcher.last_error = Some("the filesystem watch stopped".into()); + capture.health.watcher.last_error_at = Some(store::now_rfc3339()); + capture.health.watcher.consecutive_failures += 1; + capture.write_health(); + debouncer.stop(); + return Ok(1); + }; + let now = Instant::now(); + let mut config_changed = false; + let mut rescan = false; + let mut pending_appeared = false; + // a watched directory itself changed (replaced, recreated, + // renamed): its watch may be dead + let mut anchor_changed = false; + let mut throttled_changed = false; + match result { + Ok(events) => { + for event in events { + trace!("history watch: {:?} {:?}", event.kind, event.paths); + if event.kind.is_access() { + continue; + } + if event.need_rescan() { + rescan = true; + } + for path in &event.paths { + // the parent resolved, the final link kept: + // a tracked link is scheduled as the link + let path = normalize_target(path); + if state.is_config_file(&path) { + config_changed = true; + } + // a tracked path that did not exist is watched + // through an ancestor: something appearing on + // the way to it means the plan can move closer + if state.plan.pending.iter().any(|pending| pending.starts_with(&path)) { + pending_appeared = true; + } + if installed.iter().any(|anchor| anchor.path == path) { + anchor_changed = true; + } + // a link that appeared or changed may point + // somewhere new: the derived entries follow + if path.is_symlink() && state.relevant(&path) { + pending_appeared = true; + } + if !state.relevant(&path) { + debug!("history watch: ignoring {}", path.display()); + continue; + } + // files are scheduled, never directories: a + // held directory would hold everything in it + if path.is_dir() && !path.is_symlink() { + continue; + } + capture.schedule.note(path.clone(), now); + if capture.schedule.is_throttled(&path) { + throttled_changed = true; + } + } + } + } + Err(errors) => { + for err in errors { + capture.out.emit("error", &format!("watch error: {err}"), json!({ "message": err.to_string() })); + } + } + } + // a throttled file's unsaved changes are visible to status + // and doctor as they happen, not only after its next save + if throttled_changed { + capture.persist_schedule(); + capture.write_health(); + } + if config_changed { + match state.reload().await { + Ok(true) => { + installed = match install(&mut debouncer, &installed, &state.plan.anchors, &mut capture) { + Ok(installed) => installed, + Err(err) => { + stop_after_install_failure(&mut capture, &state.tracked, &err).await; + debouncer.stop(); + return Ok(1); + } + }; + // the timing settings may have changed with it + let fresh = Intervals::from_settings(&Settings::get()); + if fresh.limits != *capture.schedule.limits() { + capture.schedule.set_limits(fresh.limits.clone()); + } + if fresh.reconcile != intervals.reconcile { + next_reconcile = fresh.reconcile.map(|every| tokio::time::Instant::now() + every); + } + intervals = fresh; + capture.out.emit( + "replan", + &format!("configuration changed; watching {} anchor(s)", installed.len()), + json!({ "anchors": installed.len() }), + ); + // a path the new configuration no longer + // autosaves (excluded, untracked, switched to + // manual saving) leaves the schedule: no capture + // from now on holds it or carries its old + // version forward as if it were still eligible. + // A path that is missing right now (a symlink + // target between two versions, say) keeps its + // throttling while something still declares it; + // one nothing declares any more leaves like any + // other + prune_schedule(&mut capture, &state); + // the configuration that changed is what this + // capture is for: never held back + let config_dir = state.config_dir.clone(); + let held: Vec = capture + .schedule + .held_paths(now) + .into_iter() + .filter(|path| !path.starts_with(&config_dir)) + .collect(); + if capture.attempt(&state.tracked, "configuration changed", &held) == Attempt::Done { + for path in capture.schedule.due_paths(now).into_iter().chain( + capture + .schedule + .held_paths(now) + .into_iter() + .filter(|path| path.starts_with(&config_dir)), + ) { + capture.schedule.saved(&path, now); + } + capture.schedule.prune(now); + capture.persist_schedule(); + } + capture.health.watcher.last_reconcile = Some(store::now_rfc3339()); + capture.write_health(); + } + Ok(false) => { + // what is still pending is saved under the + // set that was in force, like any stop + capture.out.emit("disabled", "history was disabled; stopping", json!({})); + finish(&mut capture, &state.tracked).await; + debouncer.stop(); + return Ok(0); + } + Err(err) => capture.out.emit( + "error", + &format!("configuration could not be reloaded; keeping the previous tracked set: {err:#}"), + json!({ "message": format!("{err:#}") }), + ), + } + } else if rescan { + // the backend lost track: every watch is made anew + if let Ok(true) = state.reload().await { + installed = match reinstall(&mut debouncer, &installed, &state.plan.anchors, &mut capture) { + Ok(installed) => installed, + Err(err) => { + stop_after_install_failure(&mut capture, &state.tracked, &err).await; + debouncer.stop(); + return Ok(1); + } + }; + prune_schedule(&mut capture, &state); + } + capture.reconcile(&state.tracked, "rescan"); + capture.health.watcher.last_reconcile = Some(store::now_rfc3339()); + capture.write_health(); + } else if (pending_appeared || anchor_changed) + && let Ok(true) = state.reload().await + { + // a replaced directory keeps its path but not its + // watch: an anchor that changed is watched anew + installed = match if anchor_changed { + reinstall(&mut debouncer, &installed, &state.plan.anchors, &mut capture) + } else { + install(&mut debouncer, &installed, &state.plan.anchors, &mut capture) + } { + Ok(installed) => installed, + Err(err) => { + stop_after_install_failure(&mut capture, &state.tracked, &err).await; + debouncer.stop(); + return Ok(1); + } + }; + // what the new set no longer covers (a link's old + // target, say) leaves the schedule + prune_schedule(&mut capture, &state); + // an edit that landed while the watches were being + // remade is saved now, not at the next reconciliation + if anchor_changed { + capture.reconcile(&state.tracked, "watches reinstalled"); + capture.health.watcher.last_reconcile = Some(store::now_rfc3339()); + } + capture.out.emit( + "replan", + &format!("a tracked path appeared; watching {} anchor(s)", installed.len()), + json!({ "anchors": installed.len(), "pending": state.plan.pending.len() }), + ); + capture.write_health(); + } + } + _ = flush => { + let now = Instant::now(); + let due = capture.schedule.due_paths(now); + let retrying = capture.retry_due().is_some_and(|at| at <= now); + if !due.is_empty() || retrying { + let held = capture.schedule.held_paths(now); + let reason = if due.is_empty() { + "retry".to_string() + } else { + describe(&due) + }; + let done = capture.attempt(&state.tracked, &reason, &held) == Attempt::Done; + if done { + for path in &due { + match capture.schedule.saved(path, now) { + Adjustment::Stretched => { + let interval = capture.schedule.get(path).map(|s| s.interval).unwrap_or_default(); + capture.out.emit( + "throttled", + &format!( + "{} keeps changing; saving it every {} now (up to {}). Exclude it with `mise bootstrap dotfiles exclude '{}'` if it is a log, cache, or database, or track it with `--no-autosave` and save it explicitly", + display_path(path), + humantime(interval), + humantime(capture.schedule.limits().max), + display_path(path) + ), + json!({ "path": display_path(path), "interval_secs": interval.as_secs() }), + ); + } + Adjustment::Reset => capture.out.emit( + "settled", + &format!("{} settled; saving it promptly again", display_path(path)), + json!({ "path": display_path(path) }), + ), + Adjustment::Unchanged => {} + } + } + capture.schedule.prune(now); + capture.persist_schedule(); + capture.write_health(); + } + } + } + _ = reconcile => { + if let Some(every) = intervals.reconcile { + next_reconcile = Some(tokio::time::Instant::now() + every); + } + // the tracked set and every watch are made anew: a pending + // path that appeared is watched, a replaced directory's dead + // watch is replaced, and what the set no longer covers + // leaves the schedule + if let Ok(true) = state.reload().await { + installed = match reinstall(&mut debouncer, &installed, &state.plan.anchors, &mut capture) { + Ok(installed) => installed, + Err(err) => { + stop_after_install_failure(&mut capture, &state.tracked, &err).await; + debouncer.stop(); + return Ok(1); + } + }; + prune_schedule(&mut capture, &state); + } + capture.reconcile(&state.tracked, "reconcile"); + capture.health.watcher.last_reconcile = Some(store::now_rfc3339()); + capture.write_health(); + } + _ = shutdown.wait() => { + finish(&mut capture, &state.tracked).await; + break; + } + } + } + debouncer.stop(); + Ok(0) +} + +/// The final capture before the process ends: a full capture, not only the +/// due paths (a change still inside the coalescing window has not reached +/// the scheduler yet, and a throttled file's final state is saved now). The +/// backoff does not apply, and a running operation is given a moment. +/// The watches could not be re-installed after a replan: what is pending is +/// saved and the failure recorded before the process exits, so the service +/// restarts it and status says why. +async fn stop_after_install_failure( + capture: &mut Capture, + tracked: &TrackedSet, + err: &eyre::Report, +) { + capture.out.emit( + "error", + &format!( + "the watches could not be re-installed; stopping so the service restarts it: {err:#}" + ), + json!({ "message": format!("{err:#}") }), + ); + finish(capture, tracked).await; + // recorded after the final capture, which would clear it + capture.health.watcher.last_error = + Some(format!("the watches could not be re-installed: {err:#}")); + capture.health.watcher.last_error_at = Some(store::now_rfc3339()); + capture.health.watcher.consecutive_failures += 1; + capture.write_health(); +} + +async fn finish(capture: &mut Capture, tracked: &TrackedSet) { + // a full capture, not only the due paths: a change still + // inside the coalescing window has not reached the scheduler + // yet, and a throttled file's final state is saved now. The + // backoff does not apply, and a running operation is given a + // moment to finish + let now = Instant::now(); + capture.retry_at = None; + let mut outcome = capture.attempt(tracked, "shutdown", &[]); + for _ in 0..SHUTDOWN_RETRIES { + if outcome != Attempt::Deferred { + break; + } + tokio::time::sleep(SHUTDOWN_RETRY_EVERY).await; + capture.retry_at = None; + outcome = capture.attempt(tracked, "shutdown", &[]); + } + if outcome == Attempt::Done { + capture.schedule.clear_pending(now); + } else { + let pending = + capture.schedule.held_paths(now).len() + capture.schedule.due_paths(now).len(); + capture.out.emit( + "unsaved", + &format!("stopping with {pending} pending path(s) unsaved; the next start saves them"), + json!({ "pending": pending }), + ); + } + capture.persist_schedule(); + capture.out.emit("stopped", "stopping", json!({})); + capture.write_health(); +} + +fn describe(paths: &[PathBuf]) -> String { + let mut names: Vec = paths.iter().map(display_path).collect(); + names.sort(); + let extra = names.len().saturating_sub(3); + names.truncate(3); + if extra > 0 { + format!("{} +{extra} more changed", names.join(", ")) + } else { + format!("{} changed", names.join(", ")) + } +} + +pub(crate) fn humantime(duration: Duration) -> String { + let secs = duration.as_secs(); + if secs >= 3600 { + format!("{}h", secs / 3600) + } else if secs >= 60 { + format!("{}m", secs / 60) + } else { + format!("{secs}s") + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Intervals { + limits: Limits, + reconcile: Option, +} + +impl Intervals { + fn from_settings(settings: &Settings) -> Self { + let parse = |name: &str, value: &str, default: Duration| { + crate::duration::parse_duration(value).unwrap_or_else(|err| { + warn!("history.watch.{name}: {err}; using {default:?}"); + default + }) + }; + let reconcile = parse( + "reconcile", + &settings.history.watch.reconcile, + Duration::from_secs(600), + ); + Self { + limits: Limits { + base: parse( + "debounce", + &settings.history.watch.debounce, + Duration::from_secs(2), + ), + max: parse( + "max_interval", + &settings.history.watch.max_interval, + Duration::from_secs(24 * 3600), + ), + }, + reconcile: (!reconcile.is_zero()).then_some(reconcile), + } + } +} + +/// The tracked set, its watch plan, and the filters applied to events. +struct State { + /// The declared set, what captures walk. + tracked: TrackedSet, + /// The declared set plus derived entries, what is watched. + watched: TrackedSet, + plan: WatchPlan, + exclude: ExcludeSet, + hard: Vec, + config_dir: PathBuf, +} + +impl State { + async fn load() -> Result { + let tracked = TrackedSet::effective().await?; + Self::from_tracked(tracked) + } + + fn from_tracked(tracked: TrackedSet) -> Result { + let exclude = tracked.exclude_set()?; + let watched = watched_set(&tracked)?; + let plan = build_plan(&watched); + Ok(Self { + tracked, + watched, + plan, + exclude, + hard: hard_exclusions(), + config_dir: normalize(&tracked::global_config_dir()), + }) + } + + /// Reloads the configuration; `Ok(false)` when history was disabled. + async fn reload(&mut self) -> Result { + Config::reset().await?; + if !Settings::get().history.enabled { + return Ok(false); + } + let tracked = TrackedSet::effective().await?; + *self = Self::from_tracked(tracked)?; + Ok(true) + } + + /// A change to a mise configuration file: the tracked set and the + /// settings may differ now. + fn is_config_file(&self, path: &Path) -> bool { + path.starts_with(&self.config_dir) + && (path.extension().is_some_and(|ext| ext == "toml") + || path + .components() + .any(|component| component.as_os_str() == "conf.d")) + } + + /// Whether a change to `path` is one the watcher saves: under a + /// declared or derived entry (a symlink target inside the home + /// directory), autosaved, and not excluded. + fn relevant(&self, path: &Path) -> bool { + if self.hard.iter().any(|dir| path.starts_with(dir)) { + return false; + } + if path + .components() + .any(|component| component.as_os_str() == ".git") + { + return false; + } + if self.exclude.is_match(path) { + return false; + } + match self.watched.entry_for(path) { + Some(entry) => entry.policy.autosave, + None => false, + } + } + + /// Whether a path that does not exist right now may still be one the + /// watcher saves once it is back: under a declared autosave entry, or + /// where a tracked symlink (through any links on the way) points, its + /// target between two versions, say. A path nothing declares for + /// automatic saving any more is not kept for being missing. + fn may_cover_missing(&self, path: &Path) -> bool { + if self.hard.iter().any(|dir| path.starts_with(dir)) || self.exclude.is_match(path) { + return false; + } + if self + .tracked + .entry_for(path) + .is_some_and(|entry| entry.policy.autosave) + { + return true; + } + self.watched.entries.iter().any(|entry| { + entry.policy.autosave + && entry.path.is_symlink() + && tracked::link_target(&entry.path).is_some_and(|target| path.starts_with(target)) + }) + } +} + +/// The set the watcher plans and filters by: the declared entries plus the +/// derived ones the walk discovers (targets of tracked symlinks). +fn watched_set(tracked: &TrackedSet) -> Result { + let walk = tracked.walk()?; + let mut watched = tracked.clone(); + watched.entries = walk.entries; + Ok(watched) +} + +fn build_plan(tracked: &TrackedSet) -> WatchPlan { + let paths = tracked + .entries + .iter() + .filter(|entry| entry.policy.autosave) + .map(|entry| { + let kind = match std::fs::symlink_metadata(&entry.path) { + Ok(meta) if meta.is_dir() => PathKind::Directory, + Ok(_) => PathKind::File, + Err(_) => PathKind::Missing, + }; + (entry.path.clone(), kind) + }); + WatchPlan::build(paths, |path| { + path.ancestors() + .skip(1) + .find(|ancestor| ancestor.is_dir()) + .map(Path::to_path_buf) + }) +} + +/// Installs the plan's anchors, removing the ones no longer wanted. +/// Returns the anchors now installed. +/// Drops from the schedule what the tracked set no longer covers (excluded, +/// untracked, switched to manual saving, a link's old target): no capture +/// from now on holds it or carries its old version forward. A path that is +/// missing right now keeps its throttling while something still declares +/// it. +fn prune_schedule(capture: &mut Capture, state: &State) { + capture + .schedule + .retain(|path| state.relevant(path) || (!path.exists() && state.may_cover_missing(path))); + capture.persist_schedule(); +} + +/// Every watch anew: the one on a directory that was replaced or recreated +/// keeps its path but is dead, and only a fresh watch on the new inode +/// delivers events again. +fn reinstall( + debouncer: &mut Debouncer, + installed: &[Anchor], + wanted: &[Anchor], + capture: &mut Capture, +) -> Result> { + for anchor in installed { + if let Err(err) = debouncer.unwatch(&anchor.path) { + debug!("history watch: unwatch {}: {err}", anchor.path.display()); + } + } + install(debouncer, &[], wanted, capture) +} + +fn install( + debouncer: &mut Debouncer, + installed: &[Anchor], + wanted: &[Anchor], + capture: &mut Capture, +) -> Result> { + let mut current: Vec = vec![]; + capture.health.watcher.degraded.clear(); + for anchor in installed { + if wanted.contains(anchor) { + current.push(anchor.clone()); + } else if let Err(err) = debouncer.unwatch(&anchor.path) { + debug!("history watch: unwatch {}: {err}", anchor.path.display()); + } + } + for anchor in wanted { + if current.contains(anchor) { + continue; + } + let mode = match anchor.mode { + Mode::Recursive => RecursiveMode::Recursive, + Mode::Flat => RecursiveMode::NonRecursive, + }; + match debouncer.watch(&anchor.path, mode) { + Ok(()) => current.push(anchor.clone()), + Err(err) if matches!(err.kind, notify::ErrorKind::MaxFilesWatch) => { + if current.is_empty() { + bail!( + "cannot watch {}: the system's watch limit is reached (on Linux raise fs.inotify.max_user_watches)", + display_path(&anchor.path) + ); + } + let message = format!( + "cannot watch {}: the system's watch limit is reached; reconciliation still saves it (on Linux raise fs.inotify.max_user_watches)", + display_path(&anchor.path) + ); + capture.health.watcher.degraded.push(message.clone()); + capture.out.emit( + "degraded", + &message, + json!({ "path": display_path(&anchor.path) }), + ); + } + Err(err) => { + let message = format!( + "cannot watch {}: {err}; reconciliation still saves it", + display_path(&anchor.path) + ); + capture.health.watcher.degraded.push(message.clone()); + capture.out.emit( + "degraded", + &message, + json!({ "path": display_path(&anchor.path), "message": err.to_string() }), + ); + } + } + } + // nothing watched while something should be: no event would ever + // arrive, so the caller stops (and the service restarts it) instead of + // running blind until a reconciliation that may be disabled + if current.is_empty() && !wanted.is_empty() { + bail!("no watch could be installed for the tracked set"); + } + Ok(current) +} + +/// Captures with the operation lock respected, failures backed off, the +/// per-path schedule applied, and health persisted. +struct Capture { + store: Store, + out: Output, + schedule: Schedule, + health: Health, + backoff: Duration, + retry_at: Option, + /// Why the last attempt did not run, while a retry is pending. + retry_kind: Option, +} + +impl Capture { + fn new(store: Store, out: Output, limits: Limits) -> Self { + let mut schedule = Schedule::new(limits); + let persisted: PersistedSchedule = + std::fs::read_to_string(schedule_path_in(store.state_dir())) + .ok() + .and_then(|text| serde_json::from_str(&text).ok()) + .unwrap_or_default(); + let now = Instant::now(); + let now_epoch = epoch_secs(); + schedule.restore(&persisted, now, now_epoch); + // a throttled file rewritten while the watcher was down has a change + // pending: held until its next save is due, like any other + for (path, record) in &persisted.paths { + let path = PathBuf::from(path); + let Some(saved) = record.saved_epoch_secs else { + continue; + }; + let changed_since = std::fs::symlink_metadata(&path) + .and_then(|meta| meta.modified()) + .ok() + .and_then(|modified| modified.duration_since(std::time::UNIX_EPOCH).ok()) + // not strictly after: both are whole seconds, and a change + // in the same second as the save must count as pending (a + // false positive only holds the file until its save is due) + .is_some_and(|modified| modified.as_secs() >= saved); + if changed_since && schedule.get(&path).is_some_and(|s| !s.pending()) { + schedule.mark_pending(path, now); + } + } + let health = health::read(store.state_dir()).unwrap_or_default(); + Self { + store, + out, + schedule, + health, + backoff: BACKOFF_MIN, + retry_at: None, + retry_kind: None, + } + } + + /// When a deferred or failed capture is retried, if one is pending. + fn retry_due(&self) -> Option { + self.retry_kind.and(self.retry_at) + } + + /// A whole-set capture (a reconcile or rescan) that respects the + /// schedule: held paths are carried forward, and the paths that were due + /// count as saved so they are not saved again at their own deadline. + fn reconcile(&mut self, tracked: &TrackedSet, reason: &str) -> Attempt { + let now = Instant::now(); + let held = self.schedule.held_paths(now); + let due = self.schedule.due_paths(now); + let outcome = self.attempt(tracked, reason, &held); + if outcome == Attempt::Done { + for path in &due { + self.schedule.saved(path, now); + } + self.schedule.prune(now); + self.persist_schedule(); + } + outcome + } + + /// A flush deadline no earlier than the current backoff allows. + fn not_before(&self, at: Instant) -> Instant { + match self.retry_at { + Some(retry) if retry > at => retry, + _ => at, + } + } + + /// Saves a checkpoint of the tracked set, with `held` paths carried + /// forward from the newest checkpoint instead of read live. Returns + /// whether the attempt ran (a deferred or failed attempt leaves its + /// paths pending). + fn attempt(&mut self, tracked: &TrackedSet, reason: &str, held: &[PathBuf]) -> Attempt { + if let Some(retry) = self.retry_at + && Instant::now() < retry + { + return self.retry_kind.unwrap_or(Attempt::Failed); + } + let operation = + match LockFile::new(&store::operation_lock_in(self.store.state_dir())).try_lock() { + Ok(Some(lock)) => lock, + Ok(None) => { + self.out.emit( + "deferred", + "another history operation is running; saving afterwards", + json!({ "reason": reason }), + ); + self.retry_at = Some(Instant::now() + BACKOFF_MIN); + self.retry_kind = Some(Attempt::Deferred); + return Attempt::Deferred; + } + Err(err) => { + self.fail(reason, &format!("{err:#}")); + return Attempt::Failed; + } + }; + let mut draft = Draft::new(Trigger::Edit); + draft.held = held.to_vec(); + let result = self.store.attempt(tracked, draft); + drop(operation); + match result { + Ok(Outcome::Created(entry)) => { + self.recovered(); + self.health.watcher.last_capture = Some(store::now_rfc3339()); + self.out.emit( + "captured", + &format!( + "saved checkpoint {} ({reason}): {}", + entry.id, entry.checkpoint.description + ), + json!({ "id": entry.id, "uuid": entry.checkpoint.uuid, "description": entry.checkpoint.description, "reason": reason }), + ); + self.retry_kind = None; + Attempt::Done + } + Ok(Outcome::Unchanged) => { + self.recovered(); + self.health.watcher.last_capture = Some(store::now_rfc3339()); + self.out.emit( + "unchanged", + &format!("nothing to save ({reason})"), + json!({ "reason": reason }), + ); + self.retry_kind = None; + Attempt::Done + } + Ok(Outcome::Unavailable(message)) => { + self.fail(reason, &message); + Attempt::Failed + } + Err(err) => { + self.fail(reason, &format!("{err:#}")); + Attempt::Failed + } + } + } + + fn fail(&mut self, reason: &str, message: &str) { + self.out.emit( + "error", + &format!( + "could not save ({reason}): {message}; retrying in {:?}", + self.backoff + ), + json!({ "reason": reason, "message": message, "retry_in_secs": self.backoff.as_secs() }), + ); + self.retry_at = Some(Instant::now() + self.backoff); + self.retry_kind = Some(Attempt::Failed); + self.backoff = (self.backoff * 2).min(BACKOFF_MAX); + self.health.watcher.last_error = Some(message.to_string()); + self.health.watcher.last_error_at = Some(store::now_rfc3339()); + self.health.watcher.consecutive_failures += 1; + self.write_health(); + } + + fn recovered(&mut self) { + self.backoff = BACKOFF_MIN; + self.retry_at = None; + self.retry_kind = None; + self.health.watcher.last_error = None; + self.health.watcher.last_error_at = None; + self.health.watcher.consecutive_failures = 0; + } + + fn persist_schedule(&self) { + let persisted = self.schedule.persist(Instant::now(), epoch_secs()); + let path = schedule_path_in(self.store.state_dir()); + if let Err(err) = store::write_json(&path, &persisted) { + debug!("history watch: could not write {}: {err}", path.display()); + } + // what `paths --noisy` lists + let mut record = NoisyRecord::default(); + for (path, schedule) in self.schedule.throttled() { + record.paths.insert( + display_path(&path), + NoisyPath { + interval_secs: schedule.interval.as_secs(), + pending_changes: schedule.changes, + last_seen: store::now_rfc3339(), + }, + ); + } + let noisy = noisy_path_in(self.store.state_dir()); + if let Err(err) = noise::write(&noisy, &record) { + debug!("history watch: could not write {}: {err}", noisy.display()); + } + } + + fn write_health(&mut self) { + let now = Instant::now(); + self.health.throttled = self + .schedule + .throttled() + .into_iter() + .map(|(path, schedule)| ThrottledPath { + path: display_path(&path), + interval_secs: schedule.interval.as_secs(), + last_saved: schedule + .last_saved + .map(|saved| rfc3339_ago(now.saturating_duration_since(saved))), + pending_changes: schedule.changes, + heavy: schedule.interval >= schedule::HEAVY_INTERVAL, + }) + .collect(); + if let Err(err) = health::write(self.store.state_dir(), &mut self.health) { + debug!("history watch: could not write health: {err}"); + } + } +} + +fn epoch_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +fn rfc3339_ago(ago: Duration) -> String { + let at = chrono::Utc::now() - chrono::Duration::from_std(ago).unwrap_or_default(); + at.to_rfc3339_opts(chrono::SecondsFormat::Secs, true) +} + +#[derive(Clone, Copy)] +struct Output { + json: bool, +} + +impl Output { + fn emit(&self, event: &str, message: &str, mut fields: serde_json::Value) { + if self.json { + if let Some(object) = fields.as_object_mut() { + object.insert("event".into(), json!(event)); + object.insert("message".into(), json!(message)); + object.insert("at".into(), json!(store::now_rfc3339())); + } + use std::io::Write; + let mut stdout = std::io::stdout().lock(); + let _ = writeln!(stdout, "{fields}"); + let _ = stdout.flush(); + } else { + match event { + "error" | "degraded" => warn!("history watch: {message}"), + "unchanged" | "deferred" => debug!("history watch: {message}"), + _ => info!("history watch: {message}"), + } + } + } +} + +/// The lock a running watcher holds; `mise bootstrap dotfiles status` reads it. +pub(crate) fn watch_lock_in(state_dir: &Path) -> PathBuf { + store::store_dir_in(state_dir).join("watch.lock") +} + +pub(crate) fn noisy_path_in(state_dir: &Path) -> PathBuf { + store::store_dir_in(state_dir).join("noisy.json") +} + +pub(crate) fn schedule_path_in(state_dir: &Path) -> PathBuf { + store::store_dir_in(state_dir).join("watch-schedule.json") +} + +/// Whether a watcher currently holds the lock for this store. +pub(crate) fn is_running(state_dir: &Path) -> bool { + matches!( + LockFile::new(&watch_lock_in(state_dir)).try_lock(), + Ok(None) + ) +} + +struct Shutdown { + #[cfg(unix)] + terminate: tokio::signal::unix::Signal, + #[cfg(unix)] + hangup: tokio::signal::unix::Signal, + #[cfg(windows)] + ctrl_break: tokio::signal::windows::CtrlBreak, +} + +impl Shutdown { + fn new() -> Result { + Ok(Self { + #[cfg(unix)] + terminate: tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?, + #[cfg(unix)] + hangup: tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup())?, + #[cfg(windows)] + ctrl_break: tokio::signal::windows::ctrl_break()?, + }) + } + + async fn wait(&mut self) { + #[cfg(unix)] + { + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = self.terminate.recv() => {} + _ = self.hangup.recv() => {} + } + } + #[cfg(windows)] + { + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = self.ctrl_break.recv() => {} + } + } + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use crate::system::files::{FileMode, FilePolicy}; + use crate::system::history::tracked::{EntryKind, TrackedEntry}; + + fn state_of(tracked: TrackedSet, config_dir: PathBuf) -> State { + State { + watched: tracked.clone(), + plan: build_plan(&tracked), + exclude: tracked.exclude_set().unwrap(), + hard: vec![], + config_dir, + tracked, + } + } + + #[cfg(unix)] + #[test] + fn a_missing_path_keeps_its_schedule_only_while_something_declares_it() { + let dir = tempfile::tempdir().unwrap(); + let root = normalize(dir.path()); + let hypr = root.join("hypr"); + std::fs::create_dir_all(&hypr).unwrap(); + // a tracked link whose target, through another link, is between + // two versions + let link = root.join("link"); + std::os::unix::fs::symlink(root.join("hop"), &link).unwrap(); + std::os::unix::fs::symlink(root.join("elsewhere/target"), root.join("hop")).unwrap(); + let policy = FilePolicy::for_mode(FileMode::Track); + let mut tracked = TrackedSet { + entries: vec![ + TrackedEntry::new(hypr.clone(), EntryKind::Track, "track", policy), + TrackedEntry::new(link.clone(), EntryKind::Track, "track", policy), + ], + exclude: vec![format!("{}/hypr/plugins/**", root.display())], + invalid: vec![], + }; + let state = state_of(tracked.clone(), root.join("mise")); + assert!(state.may_cover_missing(&hypr.join("bindings.lua"))); + assert!(state.may_cover_missing(&root.join("elsewhere/target"))); + assert!(!state.may_cover_missing(&hypr.join("plugins/state.json"))); + assert!(!state.may_cover_missing(&root.join("untracked/state.json"))); + + // untracked, or switched to manual saving: nothing keeps it + tracked.entries[0].policy.autosave = false; + tracked.entries.pop(); + let state = state_of(tracked, root.join("mise")); + assert!(!state.may_cover_missing(&hypr.join("bindings.lua"))); + assert!(!state.may_cover_missing(&root.join("elsewhere/target"))); + } +} diff --git a/src/system/history/watch/schedule.rs b/src/system/history/watch/schedule.rs new file mode 100644 index 00000000000..041e1c4987f --- /dev/null +++ b/src/system/history/watch/schedule.rs @@ -0,0 +1,665 @@ +//! Per-file adaptive save scheduling. Ordinary edits save promptly: a +//! changed path is saved once it has been quiet for its settle time. +//! Sustained churn stretches that path's own interval — doubling on every +//! save that follows another one closely with several changes in between, +//! up to a maximum — so a constantly rewritten file is still saved +//! periodically but never floods the history. A busy path never delays an +//! ordinary one: every path is scheduled on its own. When a churning path +//! settles, its final state is captured after its settle time, and a +//! sustained quiet period resets the interval so a brief pause does not +//! restart aggressive saving. +//! +//! Every threshold is a named constant here; the schedule is pure and +//! clock-injected so each rule is tested without a filesystem. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; + +/// A save counts as churn — and doubles the interval — when the path +/// changed again within its settle time after the previous save (it never +/// really stopped) and at least this many changes arrived since. A person +/// saving from an editor every few seconds leaves gaps longer than the +/// settle time, so ordinary editing is never stretched. +pub(crate) const CHURN_CHANGES: u32 = 2; +/// A path quiet for this many intervals (at least `RESET_MIN`) forgets its +/// backoff. +pub(crate) const RESET_FACTOR: u32 = 4; +pub(crate) const RESET_MIN: Duration = Duration::from_secs(5 * 60); +/// The settle time of a stretched path: a fraction of its interval, never +/// below the base quiet period nor above this cap, so a settled file is +/// captured promptly whatever its interval grew to. +pub(crate) const SETTLE_DIVISOR: u32 = 8; +pub(crate) const SETTLE_MAX: Duration = Duration::from_secs(5 * 60); +/// A path whose interval reached this is reported as heavily throttled. +pub(crate) const HEAVY_INTERVAL: Duration = Duration::from_secs(60 * 60); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Limits { + /// The base quiet period (`history.watch.debounce`). + pub base: Duration, + /// The longest periodic interval (`history.watch.max_interval`). + pub max: Duration, +} + +impl Limits { + fn clamp(&self, interval: Duration) -> Duration { + interval.max(self.base).min(self.max.max(self.base)) + } +} + +/// One path's scheduling state. +#[derive(Clone, Debug)] +pub(crate) struct PathSchedule { + /// The current periodic interval. + pub interval: Duration, + /// When the path last changed (`None`: no change pending). + pub last_change: Option, + /// When the pending batch of changes began. + pub pending_since: Option, + /// Changes since the last save. + pub changes: u32, + pub last_saved: Option, +} + +impl PathSchedule { + fn new(base: Duration) -> Self { + Self { + interval: base, + last_change: None, + pending_since: None, + changes: 0, + last_saved: None, + } + } + + pub(crate) fn pending(&self) -> bool { + self.last_change.is_some() + } + + /// Quiet needed before a pending change is saved. + fn settle(&self, limits: &Limits) -> Duration { + // never below the configured quiet period, even one above the cap + (self.interval / SETTLE_DIVISOR) + .max(limits.base) + .min(SETTLE_MAX.max(limits.base)) + } + + /// When this path's pending change is due: quiet for its settle time, + /// or its interval since the batch began, whichever comes first. + pub(crate) fn due(&self, limits: &Limits) -> Option { + let last = self.last_change?; + let quiet = last + self.settle(limits); + let periodic = self.pending_since.unwrap_or(last) + self.interval; + Some(quiet.min(periodic)) + } + + fn reset_after(&self) -> Duration { + (self.interval * RESET_FACTOR).max(RESET_MIN) + } +} + +/// The persisted part: what a restart must not forget. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct PersistedSchedule { + #[serde(default)] + pub paths: BTreeMap, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub(crate) struct PersistedPath { + pub interval_secs: u64, + /// When the path was last saved (unix seconds), for the reset rule and + /// for telling a changed file from a quiet one after a restart. + #[serde(default)] + pub saved_epoch_secs: Option, + /// Changes seen since that save, and when the pending batch began and + /// last changed, so a restart neither forgets a pending save nor takes + /// it early. + #[serde(default)] + pub pending_changes: u32, + #[serde(default)] + pub pending_since_epoch_secs: Option, + #[serde(default)] + pub last_change_epoch_secs: Option, +} + +#[derive(Debug)] +pub(crate) struct Schedule { + limits: Limits, + paths: BTreeMap, +} + +impl Schedule { + /// Forgets every path `keep` rejects: one the configuration no longer + /// autosaves (excluded, untracked, switched to manual saving) must not + /// be held or carried forward by any later capture. + pub(crate) fn retain(&mut self, keep: impl Fn(&Path) -> bool) { + self.paths.retain(|path, _| keep(path)); + } +} + +/// What a save of a path reported: whether its interval changed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Adjustment { + Unchanged, + Stretched, + Reset, +} + +impl Schedule { + pub(crate) fn new(limits: Limits) -> Self { + Self { + limits, + paths: BTreeMap::new(), + } + } + + pub(crate) fn limits(&self) -> &Limits { + &self.limits + } + + /// Restores stretched intervals from a previous run, with their last + /// save and pending batch placed on this run's clock (`now` is the + /// instant that corresponds to the wall clock `now_epoch`), so the + /// schedule continues where it stopped: a path quiet longer than its + /// reset period comes back at the base interval, and a pending save is + /// neither forgotten nor taken early. + pub(crate) fn restore(&mut self, persisted: &PersistedSchedule, now: Instant, now_epoch: u64) { + let ago = |epoch: u64| { + now.checked_sub(Duration::from_secs(now_epoch.saturating_sub(epoch))) + .unwrap_or(now) + }; + for (path, record) in &persisted.paths { + let interval = self.limits.clamp(Duration::from_secs(record.interval_secs)); + if interval <= self.limits.base { + continue; + } + let mut schedule = PathSchedule::new(self.limits.base); + schedule.interval = interval; + let quiet_for = record + .saved_epoch_secs + .map(|saved| now_epoch.saturating_sub(saved)); + if quiet_for.is_some_and(|quiet| Duration::from_secs(quiet) >= schedule.reset_after()) { + continue; + } + schedule.last_saved = record.saved_epoch_secs.map(ago); + if record.pending_changes > 0 { + schedule.changes = record.pending_changes; + schedule.pending_since = + Some(record.pending_since_epoch_secs.map(ago).unwrap_or(now)); + schedule.last_change = Some(record.last_change_epoch_secs.map(ago).unwrap_or(now)); + } + self.paths.insert(PathBuf::from(path), schedule); + } + } + + /// The persisted form. Only stretched paths matter. + pub(crate) fn persist(&self, now: Instant, now_epoch: u64) -> PersistedSchedule { + let mut out = PersistedSchedule::default(); + let epoch = + |at: Instant| now_epoch.saturating_sub(now.saturating_duration_since(at).as_secs()); + for (path, schedule) in &self.paths { + if schedule.interval <= self.limits.base { + continue; + } + out.paths.insert( + path.to_string_lossy().into_owned(), + PersistedPath { + interval_secs: schedule.interval.as_secs(), + saved_epoch_secs: schedule.last_saved.map(epoch), + pending_changes: schedule.changes, + pending_since_epoch_secs: schedule.pending_since.map(epoch), + last_change_epoch_secs: schedule.last_change.map(epoch), + }, + ); + } + out + } + + /// Applies new limits (a settings change while running): every interval + /// is clamped into the new range. + pub(crate) fn set_limits(&mut self, limits: Limits) { + for schedule in self.paths.values_mut() { + schedule.interval = limits.clamp(schedule.interval); + } + self.limits = limits; + } + + /// Whether `path` is throttled (its interval is above the base). + pub(crate) fn is_throttled(&self, path: &Path) -> bool { + self.paths + .get(path) + .is_some_and(|schedule| schedule.interval > self.limits.base) + } + + /// A change to `path` was seen. + pub(crate) fn note(&mut self, path: PathBuf, now: Instant) { + let base = self.limits.base; + let schedule = self + .paths + .entry(path) + .or_insert_with(|| PathSchedule::new(base)); + // a path quiet for its reset period comes back at the base interval + if let Some(saved) = schedule.last_saved + && schedule.last_change.is_none() + && now.saturating_duration_since(saved) >= schedule.reset_after() + { + schedule.interval = base; + } + schedule.pending_since.get_or_insert(now); + schedule.last_change = Some(now); + schedule.changes += 1; + } + + /// A change to `path` may have happened (a same-second ambiguity after + /// a restart): the path is pending, without a change counted toward + /// churn detection. + pub(crate) fn mark_pending(&mut self, path: PathBuf, now: Instant) { + let base = self.limits.base; + let schedule = self + .paths + .entry(path) + .or_insert_with(|| PathSchedule::new(base)); + schedule.pending_since.get_or_insert(now); + schedule.last_change = Some(now); + } + + /// The next moment any pending path is due. + pub(crate) fn deadline(&self) -> Option { + self.paths + .values() + .filter_map(|schedule| schedule.due(&self.limits)) + .min() + } + + /// The pending paths that are due now. + pub(crate) fn due_paths(&self, now: Instant) -> Vec { + self.paths + .iter() + .filter(|(_, schedule)| schedule.due(&self.limits).is_some_and(|due| due <= now)) + .map(|(path, _)| path.clone()) + .collect() + } + + /// The pending paths that are not due yet: their live content is held + /// back from captures other paths trigger, so a capture for another + /// path cannot defeat this path's throttling. + pub(crate) fn held_paths(&self, now: Instant) -> Vec { + self.paths + .iter() + .filter(|(_, schedule)| schedule.due(&self.limits).is_some_and(|due| due > now)) + .map(|(path, _)| path.clone()) + .collect() + } + + /// Records that `path` was saved now and adapts its interval. + pub(crate) fn saved(&mut self, path: &Path, now: Instant) -> Adjustment { + let base = self.limits.base; + let Some(schedule) = self.paths.get_mut(path) else { + return Adjustment::Unchanged; + }; + let settle = schedule.settle(&self.limits); + let churn = match (schedule.last_saved, schedule.pending_since) { + (Some(previous), Some(since)) => { + since.saturating_duration_since(previous) <= settle + && schedule.changes >= CHURN_CHANGES + } + _ => false, + }; + let reset = schedule.last_saved.is_some_and(|previous| { + now.saturating_duration_since(previous) >= schedule.reset_after() + }) && schedule.interval > base; + let adjustment = if churn && schedule.interval < self.limits.max { + schedule.interval = self.limits.clamp(schedule.interval * 2); + Adjustment::Stretched + } else if reset { + schedule.interval = base; + Adjustment::Reset + } else { + Adjustment::Unchanged + }; + schedule.last_saved = Some(now); + schedule.last_change = None; + schedule.pending_since = None; + schedule.changes = 0; + adjustment + } + + /// Forgets pending changes for every path (after a full capture that + /// read everything live, such as an explicit save or a shutdown). + pub(crate) fn clear_pending(&mut self, now: Instant) { + for schedule in self.paths.values_mut() { + if schedule.pending() { + schedule.last_saved = Some(now); + schedule.last_change = None; + schedule.pending_since = None; + schedule.changes = 0; + } + } + } + + /// Stretched paths, for status and doctor. + pub(crate) fn throttled(&self) -> Vec<(PathBuf, &PathSchedule)> { + self.paths + .iter() + .filter(|(_, schedule)| schedule.interval > self.limits.base) + .map(|(path, schedule)| (path.clone(), schedule)) + .collect() + } + + pub(crate) fn get(&self, path: &Path) -> Option<&PathSchedule> { + self.paths.get(path) + } + + /// Drops paths that are neither pending nor saved within their reset + /// period (a recent save is what lets the next one be recognized as + /// churn; a stretched path past its reset period would come back at the + /// base interval anyway, so it is forgotten rather than reported). + pub(crate) fn prune(&mut self, now: Instant) { + self.paths.retain(|_, schedule| { + schedule.pending() + || schedule.last_saved.is_some_and(|saved| { + now.saturating_duration_since(saved) < schedule.reset_after() + }) + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn secs(n: u64) -> Duration { + Duration::from_secs(n) + } + + fn limits() -> Limits { + Limits { + base: secs(2), + max: secs(24 * 3600), + } + } + + #[test] + fn an_ordinary_edit_saves_after_the_base_quiet() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + schedule.note("a".into(), start); + assert_eq!(schedule.deadline(), Some(start + secs(2))); + assert!(schedule.due_paths(start + secs(1)).is_empty()); + assert_eq!( + schedule.due_paths(start + secs(2)), + vec![PathBuf::from("a")] + ); + assert_eq!( + schedule.saved(Path::new("a"), start + secs(2)), + Adjustment::Unchanged + ); + assert_eq!(schedule.get(Path::new("a")).unwrap().interval, secs(2)); + } + + #[test] + fn churn_stretches_the_interval_and_keeps_periodic_saves() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + let path = PathBuf::from("state.json"); + let mut now = start; + let mut saves = 0; + // one change per second for a long time: saves keep happening, ever + // more rarely + for tick in 0..2000u64 { + now = start + secs(tick); + schedule.note(path.clone(), now); + if schedule.due_paths(now).contains(&path) { + schedule.saved(&path, now); + saves += 1; + } + } + let interval = schedule.get(&path).unwrap().interval; + assert!(interval >= secs(256), "interval only reached {interval:?}"); + assert!(saves < 30, "{saves} saves for 2000 changes"); + // still pending and still due periodically, never later than the interval + let due = schedule.get(&path).unwrap().due(schedule.limits()).unwrap(); + assert!(due <= now + interval); + } + + #[test] + fn a_settled_file_is_captured_promptly() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + let path = PathBuf::from("state.json"); + for tick in 0..600u64 { + let now = start + secs(tick); + schedule.note(path.clone(), now); + if schedule.due_paths(now).contains(&path) { + schedule.saved(&path, now); + } + } + let interval = schedule.get(&path).unwrap().interval; + assert!(interval >= secs(64)); + // the last change settles: due after the settle time, not the interval + let last = start + secs(600); + schedule.note(path.clone(), last); + let settle = schedule.get(&path).unwrap().settle(schedule.limits()); + assert!(settle <= SETTLE_MAX && settle >= secs(2)); + assert_eq!( + schedule.get(&path).unwrap().due(schedule.limits()), + Some(last + settle) + ); + } + + #[test] + fn a_busy_path_never_delays_an_ordinary_one() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + for tick in 0..100u64 { + schedule.note("busy".into(), start + secs(tick)); + for path in schedule.due_paths(start + secs(tick)) { + schedule.saved(&path, start + secs(tick)); + } + } + let now = start + secs(100); + schedule.note("quiet".into(), now); + schedule.note("busy".into(), now); + assert_eq!( + schedule.due_paths(now + secs(2)), + vec![PathBuf::from("quiet")] + ); + assert!( + schedule + .held_paths(now + secs(2)) + .contains(&PathBuf::from("busy")) + ); + } + + #[test] + fn sustained_quiet_resets_but_a_brief_pause_does_not() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + let path = PathBuf::from("state.json"); + let mut now = start; + for tick in 0..300u64 { + now = start + secs(tick); + schedule.note(path.clone(), now); + if schedule.due_paths(now).contains(&path) { + schedule.saved(&path, now); + } + } + // drain the pending change + let settle = schedule.get(&path).unwrap().settle(schedule.limits()); + schedule.saved(&path, now + settle); + let stretched = schedule.get(&path).unwrap().interval; + assert!(stretched > secs(2)); + // a brief pause: still stretched + schedule.note(path.clone(), now + settle + secs(60)); + assert_eq!(schedule.get(&path).unwrap().interval, stretched); + schedule.saved(&path, now + settle + secs(62)); + // a long quiet: back to base on the next change + let reset_after = schedule.get(&path).unwrap().reset_after(); + schedule.note(path.clone(), now + settle + secs(62) + reset_after); + assert_eq!(schedule.get(&path).unwrap().interval, secs(2)); + } + + #[test] + fn an_editor_saving_every_few_seconds_is_never_stretched() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + let path = PathBuf::from(".zshrc"); + // a person saves from an editor every five seconds for ten minutes: + // each change settles before the next, so nothing counts as churn + for tick in (0..600u64).step_by(5) { + let now = start + secs(tick); + schedule.note(path.clone(), now); + assert_eq!(schedule.due_paths(now + secs(2)), vec![path.clone()]); + assert_eq!(schedule.saved(&path, now + secs(2)), Adjustment::Unchanged); + schedule.prune(now + secs(2)); + } + assert_eq!(schedule.get(&path).unwrap().interval, secs(2)); + } + + #[test] + fn pruning_between_saves_keeps_what_churn_detection_needs() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + let path = PathBuf::from("state.json"); + // a burst: saved, pruned, and immediately busy again + schedule.note(path.clone(), start); + schedule.note(path.clone(), start + secs(1)); + schedule.saved(&path, start + secs(2)); + schedule.prune(start + secs(2)); + assert!( + schedule.get(&path).is_some(), + "a recently saved path is kept" + ); + schedule.note(path.clone(), start + secs(3)); + schedule.note(path.clone(), start + secs(4)); + assert_eq!( + schedule.saved(&path, start + secs(5)), + Adjustment::Stretched + ); + assert_eq!(schedule.get(&path).unwrap().interval, secs(4)); + // a path saved long ago is dropped + let mut old = Schedule::new(limits()); + old.note("once".into(), start); + old.saved(Path::new("once"), start + secs(2)); + old.prune(start + secs(2) + RESET_MIN); + assert!(old.get(Path::new("once")).is_none()); + } + + #[test] + fn a_restart_continues_the_schedule_where_it_stopped() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + let path = PathBuf::from("/tmp/state.json"); + for tick in 0..300u64 { + let now = start + secs(tick); + schedule.note(path.clone(), now); + if schedule.due_paths(now).contains(&path) { + schedule.saved(&path, now); + } + } + // a change is pending and not due yet + let stopped = start + secs(300); + schedule.note(path.clone(), stopped); + let pending_before = schedule.get(&path).unwrap().changes; + let due_before = schedule.get(&path).unwrap().due(schedule.limits()).unwrap(); + assert!(due_before > stopped + secs(2)); + let persisted = schedule.persist(stopped, 1_000_000); + // ten seconds later the watcher is back: the same save is due at the + // same moment, the last save is remembered, and the path is held + let restarted_at = stopped + secs(10); + let mut restarted = Schedule::new(limits()); + restarted.restore(&persisted, restarted_at, 1_000_010); + let restored = restarted.get(&path).unwrap(); + assert_eq!(restored.changes, pending_before); + assert!(restored.last_saved.is_some()); + assert_eq!(restored.due(restarted.limits()), Some(due_before)); + assert!(restarted.held_paths(restarted_at).contains(&path)); + assert!(restarted.due_paths(restarted_at).is_empty()); + // an overdue save is due at once + let mut late = Schedule::new(limits()); + late.restore( + &persisted, + due_before + secs(1), + 1_000_000 + 1 + (due_before - stopped).as_secs(), + ); + assert!(late.due_paths(due_before + secs(1)).contains(&path)); + } + + #[test] + fn new_limits_clamp_every_interval() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + let path = PathBuf::from("state.json"); + for tick in 0..600u64 { + let now = start + secs(tick); + schedule.note(path.clone(), now); + if schedule.due_paths(now).contains(&path) { + schedule.saved(&path, now); + } + } + assert!(schedule.get(&path).unwrap().interval > secs(16)); + schedule.set_limits(Limits { + base: secs(2), + max: secs(16), + }); + assert_eq!(schedule.get(&path).unwrap().interval, secs(16)); + assert!(schedule.is_throttled(&path)); + } + + #[test] + fn a_throttled_path_past_its_reset_period_is_forgotten() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + let path = PathBuf::from("state.json"); + for tick in 0..300u64 { + let now = start + secs(tick); + schedule.note(path.clone(), now); + if schedule.due_paths(now).contains(&path) { + schedule.saved(&path, now); + } + } + let settle = schedule.get(&path).unwrap().settle(schedule.limits()); + schedule.saved(&path, start + secs(300) + settle); + let reset_after = schedule.get(&path).unwrap().reset_after(); + schedule.prune(start + secs(300) + settle + reset_after - secs(1)); + assert!(schedule.is_throttled(&path)); + schedule.prune(start + secs(300) + settle + reset_after); + assert!(schedule.get(&path).is_none()); + } + + #[test] + fn persisted_intervals_survive_a_restart_unless_quiet_long_enough() { + let start = Instant::now(); + let mut schedule = Schedule::new(limits()); + let path = PathBuf::from("/tmp/state.json"); + for tick in 0..300u64 { + let now = start + secs(tick); + schedule.note(path.clone(), now); + if schedule.due_paths(now).contains(&path) { + schedule.saved(&path, now); + } + } + let persisted = schedule.persist(start + secs(300), 1_000_000); + let record = persisted.paths.get("/tmp/state.json").unwrap(); + assert!(record.interval_secs > 2); + + let mut restarted = Schedule::new(limits()); + restarted.restore(&persisted, start + secs(360), 1_000_060); + assert_eq!( + restarted.get(&path).unwrap().interval, + secs(record.interval_secs) + ); + + let mut later = Schedule::new(limits()); + later.restore( + &persisted, + start + secs(300 + 24 * 3600), + 1_000_000 + 24 * 3600, + ); + assert!(later.get(&path).is_none()); + } +}