[PoC, do not merge] push-based auto-refresh on accessibility-tree changes

configs/default-config.toml

@@ -100,12 +100,34 @@ visible_check_enabled = false
 "Left" = "action move_mouse_relative --dx=-10 --dy=0"
 "Right" = "action move_mouse_relative --dx=10 --dy=0"
 
+[hints.auto_refresh]
+# Push-based auto-refresh of hints while hints mode is active (macOS). neru
+# watches the processes the hint scan targeted via accessibility change
+# notifications (AXObserver) and re-scans when they change, instead of relying on
+# a fixed post-action delay. Observers run only while hints mode is active, so an
+# idle neru has no background cost.
+#
+# Web/Electron coverage is best effort: it depends on the app posting
+# accessibility notifications, which Chromium/WebKit do inconsistently (often
+# nothing on scroll). Native apps, menus, and Notification Center are reliable.
+enabled = false
+debounce_ms = 150            # settle delay: wait this long after the last change before rescanning
+                             # (many apps post a notification before their tree finishes updating)
+watch_value_changed = false  # also observe kAXValueChanged on the front window (noisy)
+
 [hints.additional_ax_support]
-enable = false                              # Enable enhanced AX for Electron/Chromium/Firefox/WebKit apps
+# Wakes Electron/Chromium/Firefox accessibility trees so their hints (and change
+# notifications) work. AXManualAccessibility is attempted for every activated app
+# (a safe no-op on apps that do not implement it).
+enable = false
 additional_electron_bundles = []
 additional_chromium_bundles = []
 additional_firefox_bundles = []
 additional_webkit_bundles = []
+# When AXEnhancedUserInterface (which can relayout/move some apps) is used:
+# "whitelist" (default) = known/added browsers only; "off" = never; "all" = any
+# app whose tree does not wake.
+escalate_enhanced = "whitelist"
 
 [hints.ui]
 font_size = 10

docs/CONFIGURATION.md

@@ -426,17 +426,37 @@ y_offset = 24
 width = 320
 ```
 
+### Auto Refresh
+
+Push-based auto-refresh of hints while hints mode is active (macOS). When enabled, neru watches the processes the hint scan targeted using accessibility change notifications (`AXObserver`) and re-scans when they actually change, instead of relying on a fixed post-action delay. Observers run only while hints mode is active, so an idle neru has no background cost.
+
+Coverage for web and Electron content is best effort: it depends on the app posting accessibility notifications, which Chromium and WebKit do inconsistently (for example, often nothing on scroll). Native apps, menus, and Notification Center are reliable.
+
+| Option                | Type | Default | Description                                                                      |
+| --------------------- | ---- | ------- | -------------------------------------------------------------------------------- |
+| `enabled`             | bool | `false` | Enable push-based auto-refresh while hints mode is active                         |
+| `debounce_ms`         | int  | `80`    | Trailing coalesce/floor interval; a burst of changes collapses into one refresh  |
+| `watch_value_changed` | bool | `false` | Also observe `kAXValueChanged` on the front window (noisy; off by default)        |
+
+```toml
+[hints.auto_refresh]
+enabled = false
+debounce_ms = 80
+watch_value_changed = false
+```
+
 ### Additional AX Support
 
-Framework-specific accessibility improvements for Electron, Chromium, Firefox, and WebKit apps:
+Wakes Electron, Chromium, and Firefox accessibility trees so their hints (and auto-refresh notifications) work. `AXManualAccessibility` is attempted for every activated app (a safe no-op on apps that do not implement it, so it also covers Electron apps that are not on any list). `AXEnhancedUserInterface`, which can relayout or move some apps, is only used according to `escalate_enhanced`.
 
-| Option                        | Type  | Default | Description                  |
-| ----------------------------- | ----- | ------- | ---------------------------- |
-| `enable`                      | bool  | `false` | Enable additional AX support |
-| `additional_electron_bundles` | array | `[]`    | Bundle IDs of Electron apps  |
-| `additional_chromium_bundles` | array | `[]`    | Bundle IDs of Chromium apps  |
-| `additional_firefox_bundles`  | array | `[]`    | Bundle IDs of Firefox apps   |
-| `additional_webkit_bundles`   | array | `[]`    | Bundle IDs of WebKit apps    |
+| Option                        | Type   | Default       | Description                                                                                     |
+| ----------------------------- | ------ | ------------- | ---------------------------------------------------------------------------------------------- |
+| `enable`                      | bool   | `false`       | Enable waking accessibility trees on app activation                                            |
+| `additional_electron_bundles` | array  | `[]`          | Bundle IDs of Electron apps                                                                     |
+| `additional_chromium_bundles` | array  | `[]`          | Bundle IDs of Chromium apps                                                                     |
+| `additional_firefox_bundles`  | array  | `[]`          | Bundle IDs of Firefox apps                                                                      |
+| `additional_webkit_bundles`   | array  | `[]`          | Bundle IDs of WebKit apps                                                                       |
+| `escalate_enhanced`           | string | `"whitelist"` | When `AXEnhancedUserInterface` is used: `"whitelist"` (browsers only), `"off"`, or `"all"`      |
 
 ```toml
 [hints.additional_ax_support]
@@ -445,6 +465,7 @@ additional_electron_bundles = []
 additional_chromium_bundles = []
 additional_firefox_bundles = []
 additional_webkit_bundles = []
+escalate_enhanced = "whitelist"
 ```
 
 Find bundle IDs: `osascript -e 'id of app "Safari"'`

internal/app/app_initialization_steps.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"os"
 
 	"go.uber.org/zap"
 
@@ -18,6 +19,7 @@ import (
 	domainHint "github.com/y3owk1n/neru/internal/core/domain/hint"
 	"github.com/y3owk1n/neru/internal/core/domain/state"
 	derrors "github.com/y3owk1n/neru/internal/core/errors"
+	"github.com/y3owk1n/neru/internal/core/infra/axobserver"
 	eventtapadapter "github.com/y3owk1n/neru/internal/core/infra/eventtap"
 	ipcadapter "github.com/y3owk1n/neru/internal/core/infra/ipc"
 	"github.com/y3owk1n/neru/internal/core/infra/platform"
@@ -413,6 +415,18 @@ func initializeModeHandler(app *App) {
 		app.textInput,
 		app.systemPort,
 	)
+
+	// Wire push-based hint auto-refresh. The manager is inert until Reconcile is
+	// called (only while hints mode is active), so it is always constructed;
+	// disabling auto-refresh simply means Reconcile is never called. On non-darwin
+	// platforms the underlying observer is a no-op.
+	app.observers = axobserver.NewManager(deps.logger, axobserver.Config{
+		SelfPID:           os.Getpid(),
+		SelfBundleID:      config.BundleNeru,
+		WatchValueChanged: deps.config.Hints.AutoRefresh.WatchValueChanged,
+		OnChange:          app.modes.RequestObserverRefresh,
+	})
+	app.modes.SetObserverController(app.observers)
 }
 
 // initializeIPCController sets up the IPC controller for external communication.

internal/app/app_types.go

@@ -14,6 +14,7 @@ import (
 	"github.com/y3owk1n/neru/internal/config"
 	"github.com/y3owk1n/neru/internal/core/domain"
 	"github.com/y3owk1n/neru/internal/core/domain/state"
+	"github.com/y3owk1n/neru/internal/core/infra/axobserver"
 	"github.com/y3owk1n/neru/internal/core/ports"
 	"github.com/y3owk1n/neru/internal/ui"
 )
@@ -59,6 +60,10 @@ type App struct {
 
 	modes *modes.Handler
 
+	// observers watches the processes the hint scan targeted and drives push-based
+	// hint auto-refresh. Inert unless auto-refresh is configured on.
+	observers *axobserver.Manager
+
 	// Control channels
 	stopChan    chan struct{}
 	stopOnce    sync.Once

internal/app/components/hints/overlay_darwin.go

@@ -667,16 +667,18 @@ func (o *Overlay) drawHintsIncremental(
 		return true
 	}
 
-	// Handle structural changes (hints added/removed) using incremental C API
-	return o.drawHintsIncrementalStructural(
-		hints,
-		previousHints,
-		currentInput,
-		style,
-		previousInput,
-		previousStyle,
-		showArrow,
-	)
+	// A structural change (hints added/removed) falls back to a full redraw.
+	//
+	// The incremental structural path diffed purely by on-screen position, and only
+	// treated the update as a full replacement when *every* hint changed. A partial
+	// change that keeps most hints but swaps a few — an in-page control group being
+	// dismissed while another appears, common in Electron/web apps — took the
+	// incremental path, where overlapping or colliding positions could leave the
+	// newly appeared controls without hints. A full redraw via NeruDrawHints is
+	// atomic (it replaces the whole hint set in a single repaint with no blank
+	// frame), so it is both correct and flicker-free; the only cost is redrawing
+	// every hint, which is negligible. Returning false here routes to that path.
+	return false
 }
 
 // hintsAreStructurallyEqual checks if two hint lists have the same structure (same hints at same positions).

internal/app/lifecycle.go

@@ -193,6 +193,14 @@ func (a *App) setupAppWatcherCallbacks() {
 		a.handleAppActivation(bundleID)
 	})
 
+	// Proactively disarm observers for a quit application (dead pid), so a
+	// terminated app's observer is torn down without waiting for the next scan.
+	a.appWatcher.OnTerminate(func(_, bundleID string) {
+		if a.observers != nil {
+			a.observers.HandleAppTerminated(bundleID)
+		}
+	})
+
 	// Watch for display parameter changes (monitor unplug/plug, resolution changes)
 	a.appWatcher.OnScreenParametersChanged(func() {
 		a.handleScreenParametersChange()
@@ -427,23 +435,45 @@ func (a *App) handleAppActivation(bundleID string) {
 	}
 }
 
-// handleAdditionalAccessibility configures accessibility support for Electron/Chromium/Firefox applications.
+// handleAdditionalAccessibility wakes an application's accessibility tree so its
+// hints (and change notifications) work. AXManualAccessibility is attempted for
+// every activated app (a safe no-op on apps that do not implement it, so it
+// catches Electron apps that are not on any list), while AXEnhancedUserInterface
+// escalation — which can relayout/move native apps — is gated by the configured
+// EscalateEnhanced mode so it is never sprayed on native apps.
 func (a *App) handleAdditionalAccessibility(bundleID string, cfg *config.Config) {
-	config := cfg.Hints.AdditionalAXSupport
-
-	isElectron := electron.ShouldEnableElectronSupport(bundleID, config.AdditionalElectronBundles)
-	isChromium := electron.ShouldEnableChromiumSupport(bundleID, config.AdditionalChromiumBundles)
-	isFirefox := electron.ShouldEnableFirefoxSupport(bundleID, config.AdditionalFirefoxBundles)
-
-	if !isElectron && !isChromium && !isFirefox {
-		return
+	axCfg := cfg.Hints.AdditionalAXSupport
+
+	// Classification decides only whether AXEnhancedUserInterface may be used.
+	isElectron := electron.ShouldEnableElectronSupport(bundleID, axCfg.AdditionalElectronBundles)
+	isChromium := electron.ShouldEnableChromiumSupport(bundleID, axCfg.AdditionalChromiumBundles)
+	isFirefox := electron.ShouldEnableFirefoxSupport(bundleID, axCfg.AdditionalFirefoxBundles)
+	classified := isElectron || isChromium || isFirefox
+
+	allowEnhanced := false
+
+	switch axCfg.EscalateEnhanced {
+	case config.EscalateEnhancedAll:
+		allowEnhanced = true
+	case config.EscalateEnhancedOff:
+		allowEnhanced = false
+	default: // whitelist (and any unrecognized value): classified browsers/electron only
+		allowEnhanced = classified
 	}
 
 	go func() {
-		// Apps may need time to initialize their accessibility tree after launch.
-		// We retry a few times to ensure the accessibility attributes are successfully set.
-		// Use exponential backoff to minimize latency for fast-booting apps while
-		// still accommodating slow-booting ones.
+		// Unclassified apps get a single cheap attempt (short wait, negative-cached
+		// in EnsureAppAccessibility), so a native app is not re-probed with a full
+		// retry storm on every activation.
+		if !classified {
+			electron.EnsureAppAccessibility(bundleID, allowEnhanced, false, a.logger)
+
+			return
+		}
+
+		// Classified apps (browsers, listed Electron) may need time to initialize
+		// their accessibility tree after launch. Retry with exponential backoff to
+		// minimize latency for fast-booting apps while accommodating slow ones.
 		const (
 			maxRetries    = 5
 			initialDelay  = 100 * time.Millisecond
@@ -452,31 +482,10 @@ func (a *App) handleAdditionalAccessibility(bundleID string, cfg *config.Config)
 
 		delay := initialDelay
 		for range maxRetries {
-			allSuccess := true
-
-			if isElectron {
-				if !electron.EnsureElectronAccessibility(bundleID, a.logger) {
-					allSuccess = false
-				}
-			}
-
-			if isChromium {
-				if !electron.EnsureChromiumAccessibility(bundleID, a.logger) {
-					allSuccess = false
-				}
-			}
-
-			if isFirefox {
-				if !electron.EnsureFirefoxAccessibility(bundleID, a.logger) {
-					allSuccess = false
-				}
-			}
-
-			if allSuccess {
+			if electron.EnsureAppAccessibility(bundleID, allowEnhanced, true, a.logger) {
 				return
 			}
 
-			// Wait before retrying
 			time.Sleep(delay)
 			delay *= backoffFactor
 		}
@@ -595,6 +604,13 @@ func (a *App) Cleanup() {
 		}
 
 		a.ExitMode()
+		// Tear down push auto-refresh: stop the coordinator and close the observer
+		// (disarm all, stop+join the run-loop thread). Done after ExitMode so no
+		// refresh is scheduled mid-teardown, and before appWatcher.Stop so a
+		// terminate callback cannot race it.
+		if a.modes != nil {
+			a.modes.ShutdownAutoRefresh()
+		}
 		// Stop theme observer: nil the handler first so any in-flight KVO callback
 		// (between the async dispatch and actual observer removal) is a no-op.
 		a.stopThemeObserver()

internal/app/modes/handler.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"image"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"go.uber.org/zap"
@@ -94,7 +95,33 @@ type Handler struct {
 	shutdown                   func()
 	refreshHintsTimer          *time.Timer
 	modeSession                uint64
-	hotkeyLastKey              string
+
+	// Push-based hint auto-refresh (macOS). observers watches the processes the
+	// hint scan targeted; refreshCoordinator coalesces change notifications into
+	// refreshes and defers them while the user is mid-selection. Both are nil/
+	// inert unless auto-refresh is configured on.
+	observers          ObserverController
+	refreshCoordinator *refreshCoordinator
+	// observerScanning is true while a hint scan is running, and observerSuppressUntil
+	// (unix nanos) opens a short margin after a scan that changed nothing. Together
+	// they mute observer-driven refreshes: scanning an app's AX tree makes some apps
+	// create/destroy elements throughout the scan, and those self-induced
+	// notifications must not trigger another refresh (a flicker loop). The scanning
+	// flag covers the whole scan (a slow scan outlasts any fixed window); the post-
+	// scan margin only opens when the scan produced the same hint set as the previous
+	// one, so a scan that caught a real change stays hot to converge on the settled
+	// state instead of dropping the follow-up notifications (the source of missed
+	// refreshes). The fingerprint fields below drive that decision; all four are
+	// touched only under h.mu (the scan path and its deferred cleanup both hold it).
+	observerScanning           atomic.Bool
+	observerSuppressUntil      atomic.Int64
+	observerLastFingerprint    uint64
+	observerScanFingerprint    uint64
+	observerScanHasFingerprint bool
+	observerScanIsRefresh      bool
+	observerSettleChecks       int
+
+	hotkeyLastKey     string
 	hotkeyLastKeyTime          int64
 
 	textInput                  ports.TextInputPort
@@ -234,6 +261,20 @@ func NewHandler(
 		domain.ModeMonitorSelect: NewMonitorSelectMode(handler),
 	}
 
+	// The refresh coordinator coalesces observer-driven refreshes. It is always
+	// constructed (cheap and inert until Request is called), so RequestObserverRefresh
+	// is safe even before an observer controller is wired.
+	debounce := time.Duration(0)
+	if config != nil && config.Hints.AutoRefresh.DebounceMs > 0 {
+		debounce = time.Duration(config.Hints.AutoRefresh.DebounceMs) * time.Millisecond
+	}
+
+	handler.refreshCoordinator = newRefreshCoordinator(refreshCoordinatorConfig{
+		debounce:    debounce,
+		onRefresh:   handler.observerDrivenRefresh,
+		shouldDefer: handler.isMidSelection,
+	})
+
 	return handler
 }