automata: don't use the backtracker when it can't search any haystack

regex-automata/src/meta/wrappers.rs

@@ -208,6 +208,21 @@ impl BoundedBacktrackerEngine {
                 .configure(backtrack_config)
                 .build_from_nfa(nfa.clone())
                 .map_err(BuildError::nfa)?;
+            // If the backtracker can't even search an empty haystack, then
+            // don't use it at all. `max_haystack_len` saturates to `0` in that
+            // case, which the length check in `get` below can't distinguish
+            // from "an empty haystack is fine."
+            let nfa_state_len = engine.get_nfa().states().len();
+            let max_capacity = 8 * engine.get_config().get_visited_capacity();
+            if nfa_state_len > max_capacity {
+                debug!(
+                    "BoundedBacktracker not used because it needs at least \
+                     {:?} bits of visited capacity to search even an empty \
+                     haystack, but only has {:?} bits",
+                    nfa_state_len, max_capacity,
+                );
+                return Ok(None);
+            }
             debug!(
                 "BoundedBacktracker built (max haystack length: {:?})",
                 engine.max_haystack_len()

regex-automata/tests/meta/mod.rs

@@ -1,2 +1,3 @@
+mod regression;
 #[cfg(not(miri))]
 mod suite;

regex-automata/tests/meta/regression.rs

@@ -0,0 +1,21 @@
+use regex_automata::meta::Regex;
+
+// Tests that a regex whose NFA is too big for the bounded backtracker's
+// visited capacity doesn't panic on an empty haystack.
+//
+// See: https://github.com/rust-lang/regex/issues/1344
+#[test]
+fn backtracker_visited_capacity_too_small_empty_haystack() {
+    // The other engines are disabled so that this exercises the backtracker
+    // path rather than relying on them declining this pattern.
+    let config = Regex::config()
+        .nfa_size_limit(Some(1_000_000_000))
+        .dfa(false)
+        .hybrid(false)
+        .onepass(false);
+    let re =
+        Regex::builder().configure(config).build(r"^.{0,404600}$").unwrap();
+
+    assert!(re.is_match(""));
+    assert_eq!(Some((0, 0)), re.find("").map(|m| (m.start(), m.end())));
+}