feat(drawing): improved searching responsiveness (#55)

Author: blindFS

src/action/word_picker.rs

@@ -63,7 +63,7 @@ impl WordPicker {
         };
 
         autoreleasepool(|_| {
-            if let Some((attr_string, _)) = word_picker.get_attributed_string(None, false, "", "") {
+            if let Some((attr_string, _)) = word_picker.get_attributed_string(None, "", "") {
                 drawer.draw_attributed_string(attr_string, true);
             }
         });
@@ -75,17 +75,13 @@ impl WordPicker {
         &mut self,
         drawer: &UIDrawer,
         multi_selection_idx: Option<usize>,
-        is_searching: bool,
         label_prefix: &str,
         text_prefix: &str,
     ) {
         autoreleasepool(|_| {
-            if let Some((attr_string, matched)) = self.get_attributed_string(
-                multi_selection_idx,
-                is_searching,
-                label_prefix,
-                text_prefix,
-            ) {
+            if let Some((attr_string, matched)) =
+                self.get_attributed_string(multi_selection_idx, label_prefix, text_prefix)
+            {
                 self.matched = matched;
                 drawer.draw_attributed_string(attr_string, true);
             };
@@ -97,7 +93,6 @@ impl WordPicker {
         &self,
         width_height_ratio: f64,
         multi_selection_idx: Option<usize>,
-        is_searching: bool,
         label_prefix: &str,
         text_prefix: &str,
     ) -> (String, Vec<usize>) {
@@ -168,12 +163,10 @@ impl WordPicker {
 
         let buffer = format!(
             "<span class=\"h\">{}</span>\n{buffer}",
-            if is_searching {
-                format!("/{}", text_prefix)
-            } else if !label_prefix.is_empty() && matched.is_empty() {
-                "Press 󰁮 to go back".into()
+            if !(label_prefix.is_empty() && text_prefix.is_empty()) && matched.is_empty() {
+                "Press 󰁮 to return"
             } else {
-                "Press / to search".into()
+                "Press / to search"
             }
         );
 
@@ -202,14 +195,12 @@ impl WordPicker {
     fn get_attributed_string(
         &self,
         multi_selection_idx: Option<usize>,
-        is_searching: bool,
         label_prefix: &str,
         text_prefix: &str,
     ) -> Option<(Retained<NSMutableAttributedString>, Vec<usize>)> {
         let (html_str, matched) = self.to_string(
             self.screen_ratio,
             multi_selection_idx,
-            is_searching,
             label_prefix,
             text_prefix,
         );

src/app_engine/drawing.rs

@@ -292,7 +292,6 @@ impl AppEngine {
         word_picker.update_text_layer(
             &self.drawer,
             self.multi_selection.one_side_idx,
-            self.is_searching,
             &self.hint_prefix,
             &self.search_prefix,
         );

src/app_engine/filtering.rs

@@ -2,15 +2,18 @@ use std::collections::HashSet;
 
 use super::AppEngine;
 use crate::{
-    FilterMode, Mode,
+    AppSignal, FilterMode, Mode,
     action::perform_ocr,
+    app_engine::lifecycle::delay,
     ax_element::{ElementOfInterest, Target},
     config::RoleOfInterest,
     user_interface::{HintBox, hint_boxes_from_frames},
     util::{Frame, lower_ascii, select_range_helper},
 };
 use log::Level;
 
+const DEBOUNCE_TIMEOUT: u64 = 150;
+
 impl AppEngine {
     fn ocr_res_filtering(&mut self) {
         if self.hint_boxes.is_empty() {
@@ -230,7 +233,11 @@ impl AppEngine {
                 self.activate(Target::ChildElement);
             }
             FilterMode::WordPicking => {
-                if !self.multi_selection.is_on || self.multi_selection.one_side_idx.is_none() {
+                if !self.search_prefix.is_empty() {
+                    self.search_prefix.clear();
+                    self.draw_word_picker();
+                } else if !self.multi_selection.is_on || self.multi_selection.one_side_idx.is_none()
+                {
                     // Go back to text action menu
                     self.word_picker = None;
                     self.draw_element_menu("", RoleOfInterest::PseudoText, true);
@@ -263,26 +270,8 @@ impl AppEngine {
         };
     }
 
-    pub(super) async fn perform_filtering(&mut self, key_char: char, mode: FilterMode) {
-        if self.is_searching {
-            if key_char == '󰁮' {
-                if self.search_prefix.is_empty() {
-                    self.is_searching = false;
-                    self.set_mode(mode.to_app_mode());
-                    if self.word_picker.is_some() {
-                        self.draw_word_picker();
-                    } else {
-                        self.drawer.clear_menus();
-                    }
-                    return;
-                } else {
-                    self.search_prefix.pop();
-                }
-            } else {
-                self.search_prefix.push(key_char.to_ascii_lowercase());
-            }
-            self.drawer.draw_search_bar(&self.search_prefix, false);
-        } else if key_char == '󰁮' {
+    pub(super) async fn filter_by_hint(&mut self, key_char: char, mode: FilterMode) {
+        if key_char == '󰁮' {
             if self.hint_prefix.is_empty() {
                 self.go_back_in_filtering(mode);
                 return;
@@ -293,10 +282,45 @@ impl AppEngine {
             self.hint_prefix.push(key_char);
         }
 
-        // TODO: debounce in searching mode?
         self.check_filtering(mode).await;
     }
 
+    pub(super) async fn filter_by_search(&mut self, key_char: char, mode: FilterMode) {
+        if key_char == '󰁮' {
+            if self.search_prefix.is_empty() {
+                self.is_searching = false;
+                self.set_mode(mode.to_app_mode());
+                if self.word_picker.is_some() {
+                    self.drawer.hide_search_bar();
+                    self.draw_word_picker();
+                } else {
+                    self.drawer.clear_menus();
+                    self.update_hints();
+                }
+                return;
+            } else {
+                self.search_prefix.pop();
+            }
+        } else {
+            self.search_prefix.push(key_char.to_ascii_lowercase());
+        }
+
+        self.drawer.draw_search_bar(&self.search_prefix, false);
+
+        // Delay the heavy `check_filtering` call
+        self.search_debounce_counter = self.search_debounce_counter.wrapping_add(1);
+        let current_id = self.search_debounce_counter;
+        let sender = self.signal_sender.clone();
+        tokio::spawn(async move {
+            delay(
+                sender,
+                AppSignal::SearchDebounce(current_id, mode),
+                DEBOUNCE_TIMEOUT,
+            )
+            .await
+        });
+    }
+
     fn ready_for_unique(&self) -> bool {
         !self.is_searching
             && (self.hint_prefix.len() == self.hint_width as usize

src/app_engine/lifecycle.rs

@@ -1,6 +1,6 @@
 use super::AppEngine;
 use crate::{
-    Mode,
+    AppSignal, Mode,
     ax_element::{ElementOfInterest, ElementSignal, Target, ThreadSafeElement, traverse},
     config::{GlyphlowConfig, RoleOfInterest, VisibilityCheckingLevel},
     os_util::get_focused_window,
@@ -55,6 +55,7 @@ impl AppEngine {
         self.search_prefix.clear();
         self.is_searching = false;
         self.multi_selection.reset();
+        self.search_debounce_counter = self.search_debounce_counter.wrapping_add(1);
     }
 
     pub(super) fn notify_then_deactivate(&mut self, msg: &str, log_level: Level) {
@@ -67,11 +68,13 @@ impl AppEngine {
             Level::Trace | Level::Info => SHORT_TIMEOUT,
             Level::Debug => DEBUG_TIMEOUT,
             _ => LONG_TIMEOUT,
-        };
+        } * 1000;
         log::log!(log_level, "{msg}");
         let id = self.drawer.notify(msg);
-        let sender = self.timeout_sender.clone();
-        tokio::spawn(async move { delay(sender, id, timeout_secs).await });
+        let sender = self.signal_sender.clone();
+        tokio::spawn(
+            async move { delay(sender, AppSignal::ClearNotification(id), timeout_secs).await },
+        );
     }
 
     pub(super) fn get_app_window_info(&mut self) {
@@ -238,7 +241,9 @@ impl AppEngine {
                 self.notify("Press Enter to act.", Level::Trace);
             }
             // For internal activations like workflow action / element explorer
-            self.set_mode(Mode::Filtering);
+            if matches!(target, Target::ChildElement | Target::Custom(_)) {
+                self.set_mode(Mode::Filtering);
+            }
         } else if self.target == Target::Scrollable
             && let Some(eoi) = self.selected.as_ref()
         {
@@ -282,7 +287,7 @@ impl AppEngine {
     }
 }
 
-async fn delay(sender: Sender<usize>, id: usize, timeout_secs: u64) {
-    tokio::time::sleep(Duration::from_secs(timeout_secs)).await;
+pub async fn delay<T>(sender: Sender<T>, id: T, timeout_millis: u64) {
+    tokio::time::sleep(Duration::from_millis(timeout_millis)).await;
     let _ = sender.send(id).await;
 }

src/app_engine/mod.rs

@@ -82,7 +82,8 @@ pub struct AppEngine {
     pub(super) editing: Option<ElementOfInterest>,
     /// For editing element text values
     pub(super) temp_file: PathBuf,
-    pub(super) timeout_sender: Sender<usize>,
+    pub(super) signal_sender: Sender<AppSignal>,
+    pub(super) search_debounce_counter: usize,
     /// Special treatment for Electron based apps.
     /// Like simulate mouse clicking instead of `element.press()`
     pub(super) last_app_window_info: AppWindowInfo,
@@ -98,7 +99,7 @@ impl AppEngine {
         key_state: Arc<Mutex<KeyState>>,
         config: GlyphlowConfig,
         temp_file: PathBuf,
-        timeout_sender: Sender<usize>,
+        signal_sender: Sender<AppSignal>,
     ) -> Self {
         let mtm = MainThreadMarker::new().expect("Not on main thread");
         let screen_frames = get_screen_frames(mtm);
@@ -125,7 +126,8 @@ impl AppEngine {
             overlay_frame,
             drawer,
             config,
-            timeout_sender,
+            signal_sender,
+            search_debounce_counter: 0,
             selected: None,
             editing: None,
             temp_file,
@@ -149,13 +151,17 @@ impl AppEngine {
             AppSignal::DeActivate => {
                 self.deactivate();
             }
+            AppSignal::MenuRefresh(key_prefix) => {
+                self.menu_refresh(&key_prefix, false);
+            }
             AppSignal::RunWorkFlow(idx) => {
                 self.drawer.clear_menus();
                 self.execute_workflow(idx);
             }
-            AppSignal::MenuRefresh(key_prefix) => {
-                self.menu_refresh(&key_prefix, false);
+            AppSignal::ScrollAction(sa) => {
+                self.perform_scroll_action(sa);
             }
+            AppSignal::TextAction(ta) => self.perform_text_action(ta),
             AppSignal::ToggleMultiSelection => match self.target {
                 Target::Text | Target::ImageOCR => {
                     self.toggle_multiselection();
@@ -167,29 +173,36 @@ impl AppEngine {
                     self.notify("Multi selection only works for text.", Level::Warn);
                 }
             },
-            AppSignal::Filter(key_char, mode) => {
-                self.perform_filtering(key_char, mode).await;
+            AppSignal::HintFilter(key_char, mode) => {
+                self.filter_by_hint(key_char, mode).await;
             }
-            AppSignal::ScrollAction(sa) => {
-                self.perform_scroll_action(sa);
+            AppSignal::SearchFilter(key_char, mode) => {
+                self.filter_by_search(key_char, mode).await;
+            }
+            AppSignal::SearchDebounce(id, mode) => {
+                if self.is_searching && id == self.search_debounce_counter {
+                    self.check_filtering(mode).await;
+                }
             }
-            AppSignal::TextAction(ta) => self.perform_text_action(ta),
             AppSignal::StartSearch => {
                 if self.is_searching {
                     return;
                 }
                 self.is_searching = true;
+                self.drawer.draw_search_bar(&self.search_prefix, true);
                 if self.word_picker.is_some() {
                     self.draw_word_picker();
                 } else {
-                    self.drawer.draw_search_bar(&self.search_prefix, true);
                     self.build_search_targets();
                 }
             }
             AppSignal::FinishSearch(mode) => {
                 self.is_searching = false;
+                self.search_debounce_counter = self.search_debounce_counter.wrapping_add(1);
                 self.set_mode(mode.to_app_mode());
-                if self.word_picker.is_none() {
+                if self.word_picker.is_some() {
+                    self.drawer.hide_search_bar();
+                } else {
                     self.drawer.clear_menus();
                 }
                 self.check_filtering(mode).await;

src/key_listener.rs

@@ -55,7 +55,8 @@ pub enum AppSignal {
     // State signals
     Activate(Target),
     DeActivate,
-    Filter(char, FilterMode),
+    HintFilter(char, FilterMode),
+    SearchFilter(char, FilterMode),
     MenuRefresh(String),
     ActOnEnter,
     // Sub state signals
@@ -72,6 +73,7 @@ pub enum AppSignal {
     FrameOCR,
     StartSearch,
     FinishSearch(FilterMode),
+    SearchDebounce(usize, FilterMode),
 }
 
 #[derive(Debug, PartialEq)]
@@ -344,7 +346,7 @@ impl KeyListener {
                     self.send(AppSignal::DeActivate);
                     *state = Mode::Idle;
                 } else {
-                    self.send(AppSignal::Filter(key_char, mode));
+                    self.send(AppSignal::HintFilter(key_char, mode));
                 }
             }
         }
@@ -398,7 +400,7 @@ impl KeyListener {
                             key.to_char()
                         };
                         let key_char = if key_char == ' ' { '󱁐' } else { key_char };
-                        self.send(AppSignal::Filter(key_char, mode));
+                        self.send(AppSignal::SearchFilter(key_char, mode));
                     }
                 };
                 true