[codex] Add plain text fallback for unsupported extensions

Cargo.toml

@@ -39,7 +39,7 @@ tree-sitter-java = "0.23.5"
 tree-sitter-ruby = "0.23.1"
 tree-sitter-php = "0.23.11"
 tree-sitter-swift = { version = "0.7.0" }
-tree-sitter-c-sharp = { version = "0.23.1" }
+tree-sitter-c-sharp = { version = "=0.23.1" }
 tree-sitter-html = "0.23.2"
 tree-sitter-md = "0.3.2"
 tree-sitter-yaml = "0.6.1"

docs/probe-cli/query.md

@@ -15,6 +15,9 @@ probe query "useState($INITIAL)" ./src --language javascript
 
 # Find class definitions in Python
 probe query "class $NAME: $$$BODY" ./src --language python
+
+# Search line-oriented text files when no parser exists
+probe query "reqproof:documents" ./docs --format json
 ```
 
 ---
@@ -120,6 +123,24 @@ probe query "go $FUNC($$$ARGS)" ./src -l go
 | `-i`, `--ignore` | String[] | - | Patterns to ignore |
 | `--no-gitignore` | Boolean | false | Don't respect .gitignore |
 | `--with-context`, `--owner-context` | Boolean | false | Include owning source-block context in JSON output |
+| `--strict` | Boolean | false | Disable plain-text fallback for unsupported extensions |
+| `--text-extension` | String[] | - | Treat an extension as plain text (repeatable, with or without `.`) |
+
+### Plain-Text Fallback
+
+When query encounters a file without a supported parser and no explicit language is requested, it falls back to line-oriented text search. The pattern is treated as a literal substring, and each matching line is returned as a `node_type: "text"` result.
+
+This allows documentation and config-style files such as `.1`, `.5`, `.txt`, `.conf`, `.tex`, `.sh`, and `.json` to be searched without a separate code path:
+
+```bash
+probe query "reqproof:documents" ./docs --format json
+```
+
+Use `--strict` for AST-only behavior, or `--text-extension EXT` to force additional suffixes into plain-text mode:
+
+```bash
+probe query "reqproof:documents" ./docs --text-extension req --format json
+```
 
 ### Language Options
 

docs/probe-cli/symbols.md

@@ -16,6 +16,12 @@ probe symbols src/main.rs src/lib.rs
 
 # Include test functions
 probe symbols src/main.rs --allow-tests
+
+# Plain-text fallback for documentation/config files
+probe symbols rsync.1 --format json
+
+# Treat a custom extension as plain text
+probe symbols notes.req --text-extension req --format json
 ```
 
 ## Basic Syntax
@@ -30,6 +36,31 @@ probe symbols <FILES> [OPTIONS]
 |--------|-------------|---------|
 | `-o, --format` | Output format: `text` or `json` | `text` |
 | `--allow-tests` | Include test functions/methods | `false` |
+| `--strict` | Disable plain-text fallback for unsupported extensions | `false` |
+| `--text-extension EXT` | Treat an extension as plain text (repeatable, with or without `.`) | - |
+
+## Plain-Text Fallback
+
+Unsupported extensions fall back to line-oriented text symbols by default. Standard documentation and config-style extensions such as `.1`, `.5`, `.txt`, `.conf`, `.tex`, `.sh`, and `.json` are treated as plain text.
+
+Plain-text entries use `kind: "text"`, an empty `name`, and the source line as `signature`:
+
+```json
+[{
+  "file": "rsync.1",
+  "symbols": [
+    {
+      "name": "",
+      "kind": "text",
+      "signature": ".TH rsync 1",
+      "line": 1,
+      "end_line": 1
+    }
+  ]
+}]
+```
+
+Use `--strict` when a caller only wants AST-backed symbols and wants unsupported extensions to be reported as warnings. Use `--text-extension EXT` to force additional suffixes into plain-text mode.
 
 ## Output Formats
 

src/cli.rs

@@ -321,6 +321,14 @@ pub enum Commands {
         /// Include test functions/methods
         #[arg(long = "allow-tests")]
         allow_tests: bool,
+
+        /// Restore strict AST-only behavior for unsupported extensions
+        #[arg(long = "strict")]
+        strict: bool,
+
+        /// Treat the given extension as plain text (repeatable, with or without leading dot)
+        #[arg(long = "text-extension", value_name = "EXT")]
+        text_extensions: Vec<String>,
     },
 
     /// Search code using AST patterns for precise structural matching
@@ -380,6 +388,14 @@ pub enum Commands {
         #[arg(long = "with-context", alias = "owner-context")]
         with_context: bool,
 
+        /// Restore strict AST-only behavior for unsupported extensions
+        #[arg(long = "strict")]
+        strict: bool,
+
+        /// Treat the given extension as plain text (repeatable, with or without leading dot)
+        #[arg(long = "text-extension", value_name = "EXT")]
+        text_extensions: Vec<String>,
+
         /// Output format (default: color)
         /// Use 'json' or 'xml' for machine-readable output with structured data
         #[arg(short = 'o', long = "format", default_value = "color", value_parser = ["markdown", "plain", "json", "xml", "color", "outline-xml"])]

src/extract/symbols.rs

@@ -3,12 +3,13 @@
 //! Provides a table-of-contents view of a file's symbols (functions, structs, classes,
 //! constants, etc.) with line numbers and nesting.
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 use serde::Serialize;
 use std::collections::HashSet;
 use std::path::Path;
 use tree_sitter::Node;
 
+use crate::file_guard;
 use crate::language::{factory::get_language_impl, get_pooled_parser, return_pooled_parser};
 
 /// Maximum nesting depth for recursive symbol collection.
@@ -35,19 +36,48 @@ pub struct FileSymbols {
     pub symbols: Vec<SymbolNode>,
 }
 
+#[derive(Debug, Clone, Default)]
+pub struct SymbolOptions {
+    pub allow_tests: bool,
+    pub strict: bool,
+    pub text_extensions: Vec<String>,
+}
+
 /// Extract the symbol tree from a file.
 pub fn extract_symbols(path: &Path, allow_tests: bool) -> Result<FileSymbols> {
+    extract_symbols_with_options(
+        path,
+        &SymbolOptions {
+            allow_tests,
+            ..SymbolOptions::default()
+        },
+    )
+}
+
+/// Extract the symbol tree from a file with configurable fallback behavior.
+pub fn extract_symbols_with_options(path: &Path, options: &SymbolOptions) -> Result<FileSymbols> {
     if !path.exists() {
         return Err(anyhow::anyhow!("File does not exist: {:?}", path));
     }
 
-    let content =
-        std::fs::read_to_string(path).context(format!("Failed to read file: {path:?}"))?;
+    let content = file_guard::read_searchable_text_file(path)?;
 
     let extension = path.extension().and_then(|ext| ext.to_str()).unwrap_or("");
 
-    let language_impl = get_language_impl(extension)
-        .ok_or_else(|| anyhow::anyhow!("Unsupported file extension: {}", extension))?;
+    let language_impl = get_language_impl(extension);
+    let user_text_extension = matches_text_extension(extension, &options.text_extensions);
+    let automatic_text_extension = is_standard_text_extension(extension);
+    if user_text_extension || (!options.strict && automatic_text_extension) {
+        return Ok(extract_plain_text_symbols(path, &content));
+    }
+    if language_impl.is_none() {
+        if options.strict {
+            return Err(anyhow::anyhow!("Unsupported file extension: {}", extension));
+        }
+        return Ok(extract_plain_text_symbols(path, &content));
+    }
+
+    let language_impl = language_impl.expect("checked language implementation presence");
 
     let mut parser = get_pooled_parser(extension)
         .map_err(|_| anyhow::anyhow!("Failed to get parser for extension: {}", extension))?;
@@ -59,7 +89,13 @@ pub fn extract_symbols(path: &Path, allow_tests: bool) -> Result<FileSymbols> {
     let root = tree.root_node();
     let source = content.as_bytes();
 
-    let mut symbols = collect_symbols(&root, source, language_impl.as_ref(), allow_tests, 0);
+    let mut symbols = collect_symbols(
+        &root,
+        source,
+        language_impl.as_ref(),
+        options.allow_tests,
+        0,
+    );
     if is_c_like_extension(extension) {
         merge_recovered_c_like_functions(&mut symbols, source);
     }
@@ -72,6 +108,48 @@ pub fn extract_symbols(path: &Path, allow_tests: bool) -> Result<FileSymbols> {
     })
 }
 
+fn extract_plain_text_symbols(path: &Path, content: &str) -> FileSymbols {
+    let symbols = content
+        .lines()
+        .enumerate()
+        .map(|(idx, line)| SymbolNode {
+            name: String::new(),
+            kind: "text".to_string(),
+            signature: line.to_string(),
+            line: idx + 1,
+            end_line: idx + 1,
+            children: Vec::new(),
+        })
+        .collect();
+
+    FileSymbols {
+        file: path.to_string_lossy().to_string(),
+        symbols,
+    }
+}
+
+pub(crate) fn normalize_extension(extension: &str) -> String {
+    extension
+        .trim()
+        .trim_start_matches('.')
+        .to_ascii_lowercase()
+}
+
+pub(crate) fn is_standard_text_extension(extension: &str) -> bool {
+    matches!(
+        normalize_extension(extension).as_str(),
+        "1" | "5" | "txt" | "conf" | "tex" | "sh" | "json"
+    )
+}
+
+pub(crate) fn matches_text_extension(extension: &str, text_extensions: &[String]) -> bool {
+    let extension = normalize_extension(extension);
+    text_extensions
+        .iter()
+        .map(|ext| normalize_extension(ext))
+        .any(|ext| ext == extension)
+}
+
 /// Container node kinds that can have child symbols.
 fn is_container_node(kind: &str) -> bool {
     matches!(
@@ -127,13 +205,17 @@ fn collect_symbols(
 
         if lang.is_symbol_node(&child) {
             let semantic_child = semantic_symbol_node(&child, lang, source).unwrap_or(child);
-            let signature = lang
-                .get_symbol_signature(&child, source)
-                .unwrap_or_else(|| {
+            let signature = if let Some(signature) = lang.get_symbol_signature(&child, source) {
+                signature
+            } else if lang.allow_symbol_signature_fallback(&child) {
+                {
                     // Fallback: use the first line of the node text
                     let text = semantic_child.utf8_text(source).unwrap_or("");
                     text.lines().next().unwrap_or("").trim().to_string()
-                });
+                }
+            } else {
+                continue;
+            };
 
             let name = extract_symbol_name(&semantic_child, source);
             let kind = normalize_kind(semantic_child.kind());
@@ -700,12 +782,12 @@ fn format_symbol_list(symbols: &[SymbolNode], output: &mut String, indent: usize
 }
 
 /// Handle the `symbols` CLI command.
-pub fn handle_symbols(files: Vec<String>, format: &str, allow_tests: bool) -> Result<()> {
+pub fn handle_symbols(files: Vec<String>, format: &str, options: SymbolOptions) -> Result<()> {
     let mut all_symbols = Vec::new();
 
     for file in &files {
         let path = Path::new(file);
-        match extract_symbols(path, allow_tests) {
+        match extract_symbols_with_options(path, &options) {
             Ok(fs) => all_symbols.push(fs),
             Err(e) => eprintln!("Warning: {}: {}", file, e),
         }
@@ -1106,9 +1188,99 @@ end
 
     #[test]
     fn test_symbols_unsupported_extension() {
+        let file = create_temp_file("hello world\nreqproof:documents SW-REQ-1", "xyz");
+        let result = extract_symbols(file.path(), false).unwrap();
+
+        assert_eq!(result.symbols.len(), 2);
+        assert_eq!(result.symbols[0].kind, "text");
+        assert_eq!(result.symbols[0].signature, "hello world");
+        assert_eq!(result.symbols[0].line, 1);
+        assert_eq!(result.symbols[1].signature, "reqproof:documents SW-REQ-1");
+        assert_eq!(result.symbols[1].line, 2);
+    }
+
+    #[test]
+    fn test_symbols_strict_unsupported_extension_errors() {
         let file = create_temp_file("hello world", "xyz");
+        let result = extract_symbols_with_options(
+            file.path(),
+            &SymbolOptions {
+                strict: true,
+                ..SymbolOptions::default()
+            },
+        );
+
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_symbols_standard_text_extensions_use_text_mode() {
+        let file = create_temp_file(".TH rsync 1\n.SH NAME", "1");
+        let result = extract_symbols(file.path(), false).unwrap();
+
+        assert_eq!(result.symbols.len(), 2);
+        assert!(result.symbols.iter().all(|symbol| symbol.kind == "text"));
+        assert_eq!(result.symbols[0].signature, ".TH rsync 1");
+        assert_eq!(result.symbols[1].signature, ".SH NAME");
+    }
+
+    #[test]
+    fn test_symbols_strict_standard_text_extension_errors_without_override() {
+        let file = create_temp_file(".TH rsync 1", "1");
+        let result = extract_symbols_with_options(
+            file.path(),
+            &SymbolOptions {
+                strict: true,
+                ..SymbolOptions::default()
+            },
+        );
+
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_symbols_custom_text_extension_forces_text_mode() {
+        let file = create_temp_file("fn not_a_symbol() {}\nplain text", "rs");
+        let result = extract_symbols_with_options(
+            file.path(),
+            &SymbolOptions {
+                text_extensions: vec!["rs".to_string()],
+                ..SymbolOptions::default()
+            },
+        )
+        .unwrap();
+
+        assert_eq!(result.symbols.len(), 2);
+        assert!(result.symbols.iter().all(|symbol| symbol.kind == "text"));
+        assert_eq!(result.symbols[0].signature, "fn not_a_symbol() {}");
+    }
+
+    #[test]
+    fn test_symbols_custom_text_extension_cannot_override_hard_deny() {
+        let file = create_temp_file("pretend text in image", "png");
+        let result = extract_symbols_with_options(
+            file.path(),
+            &SymbolOptions {
+                text_extensions: vec!["png".to_string()],
+                ..SymbolOptions::default()
+            },
+        );
+
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("hard-denied"));
+    }
+
+    #[test]
+    fn test_symbols_reject_oversized_text_file() {
+        let mut file = tempfile::Builder::new().suffix(".txt").tempfile().unwrap();
+        let content =
+            vec![b'a'; crate::file_guard::MAX_SEARCHABLE_TEXT_FILE_SIZE_BYTES as usize + 1];
+        file.write_all(&content).unwrap();
+        file.flush().unwrap();
+
         let result = extract_symbols(file.path(), false);
         assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("File too large"));
     }
 
     #[test]