Skip to content

Commit b1a5b6e

Browse files
author
SqlRush
committed
Add Grep binary detection controls
1 parent 73019fe commit b1a5b6e

4 files changed

Lines changed: 99 additions & 10 deletions

File tree

docs/claude-code-go-rewrite-plan.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ test/parity/ # golden tests against TS/official behavior
218218
- 本轮补充:Grep 路径过滤现在支持 ripgrep 风格 `glob_case_insensitive`/`globCaseInsensitive`/`glob-case-insensitive`/`--glob-case-insensitive` 及 no-override aliases,让普通 `glob`/`--glob`/`-g` 按大小写不敏感方式匹配。
219219
- 本轮补充:Grep `glob`/`--glob`/`-g` 路径过滤现在支持 ripgrep 风格 `!pattern` 排除规则,可与正向 glob、逗号/空白多 pattern 和 brace alternation 组合;只有排除规则时默认包含未被排除的路径。
220220
- 本轮补充:Grep 搜索现在支持 ripgrep 风格 `text`/`--text`/`-a``no_text`/`--no-text`,可显式把二进制扩展名文件按文本读取参与匹配,也可用 no-text 覆盖 text 模式;`-a``--no-text` 都兼容 quoted semantic boolean。
221+
- 本轮补充:Grep 二进制过滤现在从扩展名跳过改为 ripgrep 风格 NUL 内容探测;默认跳过 NUL 文件,`binary`/`--binary` 会报告匹配的 NUL 文件,`text`/`--text`/`-a` 会按文本搜索 NUL 内容,`no_binary`/`--no-binary` 可覆盖 binary。
221222
- 本轮补充:Grep content 输出现在支持 ripgrep 风格 `passthru`/`passthrough`/`--passthru`/`--passthrough`,可输出被搜索文件的全部行并保留匹配行标记;启用后按官方语义覆盖 context 行数,quoted semantic boolean 同样兼容。
222223
- 本轮补充:Grep content 输出现在支持 ripgrep 风格 `trim`/`--trim``no_trim`/`--no-trim`,会删除每条已打印文本行开头的 ASCII 空白,同时保留原始匹配列号;quoted semantic boolean 同样兼容。
223224
- 本轮补充:Grep 输出现在支持 ripgrep 风格 `stats`/`--stats``no_stats`/`--no-stats`,在 content/count/匹配文件列表结果尾部追加 matches、matched lines、files searched、bytes searched/printed 和耗时统计,并在 structured content 暴露同源统计字段。

docs/first-second-parity-audit.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ Covered behavior:
141141
- `Grep` now accepts ripgrep-style case-insensitive glob filtering through `iglob`/`--iglob`, sharing positive and `!` negative glob rule semantics with `glob` while preserving structured filter metadata.
142142
- `Grep` now accepts ripgrep-style `glob_case_insensitive`/`--glob-case-insensitive` plus no-override aliases, making ordinary `glob`/`--glob`/`-g` filters match paths case-insensitively while preserving `iglob` behavior.
143143
- `Grep` glob filtering now accepts ripgrep-style negated `!pattern` rules, including combinations with positive patterns, comma/whitespace splitting, and brace-expanded globs.
144-
- `Grep` now accepts `text`/`--text`/`-a` to search binary-extension files as text while preserving the default binary-extension skip behavior; `no_text`/`--no-text` disables text mode when both are present.
144+
- `Grep` now accepts `text`/`--text`/`-a` to disable NUL-based binary filtering and search binary content as text; `no_text`/`--no-text` disables text mode when both are present.
145+
- `Grep` binary filtering now uses ripgrep-style NUL content detection instead of extension-based skipping; default search skips NUL-containing files, `binary`/`--binary` reports matching binary files, `text`/`--text`/`-a` searches them as text, and `no_binary`/`--no-binary` restores binary filtering.
145146
- `Grep` now accepts `line_regexp`/`line-regexp`/`--line-regexp`/`-x` to require whole-line matches, with fixed-string and word-regexp precedence covered.
146147
- `Grep` content output now accepts `passthru`/`passthrough`/`--passthru`/`--passthrough`, printing all searched lines while preserving matched-line markers and overriding context counts.
147148
- `Grep` content output now accepts `vimgrep`/`--vimgrep`, repeating matching lines once per match with column metadata while preserving context-line output and `-N` formatting.

internal/tools/file/search_tools.go

Lines changed: 65 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ var allowedGrepInputKeys = map[string]struct{}{
6262
"encoding": {}, "--encoding": {}, "-E": {}, "no_encoding": {}, "noEncoding": {}, "no-encoding": {}, "--no-encoding": {},
6363
"crlf": {}, "--crlf": {}, "no_crlf": {}, "noCrlf": {}, "noCRLF": {}, "no-crlf": {}, "--no-crlf": {},
6464
"null_data": {}, "nullData": {}, "null-data": {}, "--null-data": {}, "no_null_data": {}, "noNullData": {}, "no-null-data": {}, "--no-null-data": {},
65+
"binary": {}, "--binary": {}, "no_binary": {}, "noBinary": {}, "no-binary": {}, "--no-binary": {},
6566
"fixed_strings": {}, "fixedStrings": {}, "fixed-strings": {}, "--fixed-strings": {}, "-F": {}, "multiline": {}, "--multiline": {}, "multiline-dotall": {}, "--multiline-dotall": {}, "-U": {},
6667
"text": {}, "--text": {}, "-a": {}, "no_text": {}, "noText": {}, "no-text": {}, "--no-text": {},
6768
"word_regexp": {}, "wordRegexp": {}, "word-regexp": {}, "--word-regexp": {}, "-w": {},
@@ -113,6 +114,7 @@ var grepSemanticBooleanKeys = map[string]struct{}{
113114
"no_encoding": {}, "noEncoding": {}, "no-encoding": {}, "--no-encoding": {},
114115
"crlf": {}, "--crlf": {}, "no_crlf": {}, "noCrlf": {}, "noCRLF": {}, "no-crlf": {}, "--no-crlf": {},
115116
"null_data": {}, "nullData": {}, "null-data": {}, "--null-data": {}, "no_null_data": {}, "noNullData": {}, "no-null-data": {}, "--no-null-data": {},
117+
"binary": {}, "--binary": {}, "no_binary": {}, "noBinary": {}, "no-binary": {}, "--no-binary": {},
116118
"fixed_strings": {}, "fixedStrings": {}, "fixed-strings": {}, "--fixed-strings": {}, "-F": {}, "multiline": {}, "--multiline": {}, "multiline-dotall": {}, "--multiline-dotall": {}, "-U": {},
117119
"glob_case_insensitive": {}, "globCaseInsensitive": {}, "glob-case-insensitive": {}, "--glob-case-insensitive": {}, "no_glob_case_insensitive": {}, "noGlobCaseInsensitive": {}, "no-glob-case-insensitive": {}, "--no-glob-case-insensitive": {},
118120
"text": {}, "--text": {}, "-a": {}, "no_text": {}, "noText": {}, "no-text": {}, "--no-text": {},
@@ -332,6 +334,12 @@ type grepInput struct {
332334
NoNullDataAlt bool `json:"noNullData,omitempty"`
333335
NoNullDataDash bool `json:"no-null-data,omitempty"`
334336
LongNoNullData bool `json:"--no-null-data,omitempty"`
337+
Binary bool `json:"binary,omitempty"`
338+
LongBinary bool `json:"--binary,omitempty"`
339+
NoBinary bool `json:"no_binary,omitempty"`
340+
NoBinaryAlt bool `json:"noBinary,omitempty"`
341+
NoBinaryDash bool `json:"no-binary,omitempty"`
342+
LongNoBinary bool `json:"--no-binary,omitempty"`
335343
FixedStrings bool `json:"fixed_strings,omitempty"`
336344
FixedStringsAlt bool `json:"fixedStrings,omitempty"`
337345
FixedStringsDash bool `json:"fixed-strings,omitempty"`
@@ -548,6 +556,7 @@ type grepOptions struct {
548556
IncludeZero bool
549557
ColumnNumbers bool
550558
Text bool
559+
Binary bool
551560
Encoding string
552561
SortMode string
553562
SortReverse bool
@@ -815,6 +824,12 @@ func NewGrepTool() tool.Tool {
815824
"noNullData": map[string]any{"type": "boolean"},
816825
"no-null-data": map[string]any{"type": "boolean"},
817826
"--no-null-data": map[string]any{"type": "boolean"},
827+
"binary": map[string]any{"type": "boolean"},
828+
"--binary": map[string]any{"type": "boolean"},
829+
"no_binary": map[string]any{"type": "boolean"},
830+
"noBinary": map[string]any{"type": "boolean"},
831+
"no-binary": map[string]any{"type": "boolean"},
832+
"--no-binary": map[string]any{"type": "boolean"},
818833
"fixed_strings": map[string]any{"type": "boolean"},
819834
"fixedStrings": map[string]any{"type": "boolean"},
820835
"fixed-strings": map[string]any{"type": "boolean"},
@@ -990,7 +1005,7 @@ func NewGrepTool() tool.Tool {
9901005
},
9911006
},
9921007
PromptFunc: func(tool.PromptContext) (string, error) {
993-
return "Searches text files under path using a regular expression or fixed string. pattern is the canonical search expression; regex/regexp/--regexp/-e are accepted aliases, and pattern_file/--file/-f can read one pattern per line from a file. output_mode may be files, files_with_matches, files_without_matches, content, or count; glob/-g/--glob, iglob/--iglob, type/-t/--type, and type_not/-T/--type-not optionally filter file paths. glob and iglob accept whitespace/comma-separated patterns, negation, and brace alternation; glob_case_insensitive/--glob-case-insensitive makes glob patterns ignore case. content mode supports context, before_context, after_context, -C, -B, -A, -n/--line-number and -N/--no-line-number line-number control, --column column-number output, byte_offset/--byte-offset/-b byte offset output, -H/--with-filename and -I/--no-filename filename prefix control, heading/--heading grouped file headings, path_separator/--path-separator display path separator control, null/--null NUL path terminators/separators, field_match_separator/--field-match-separator and field_context_separator/--field-context-separator output field separators, context_separator/--context-separator and no_context_separator/--no-context-separator context group separator control, offset, head_limit pagination, max_count/-m per-file match limiting, max_columns/--max-columns long-line omission, --max-columns-preview long-line previews, replace/--replace/-r display-only replacement, only_matching/-o/--only-matching matched-text output, vimgrep/--vimgrep per-match line output, passthru/--passthru/--passthrough all-line output, trim/--trim leading-whitespace trimming, stats/--stats aggregate statistics, json/--json NDJSON events, quiet/--quiet/-q output suppression, and hidden/--hidden or no_hidden/--no-hidden hidden file traversal control. Use files/--files to list files that would be searched without requiring pattern, files_with_matches or -l to list files with matches, files_without_match to list files without matches, and count/--count/-c for count mode. Count mode supports count_matches/--count-matches for occurrence counts and include_zero/--include-zero to include zero-count files. Use max_depth/--max-depth/-d to limit directory descent, max_filesize/--max-filesize with optional K/M/G suffix to skip larger files, follow/--follow/-L or no_follow/--no-follow to control symlink traversal, and sort/--sort or sortr/--sortr with path or modified to control result ordering; --sort-files is accepted as a path-sort alias. Use fixed_strings/-F/--fixed-strings for literal matching, encoding/--encoding/-E to choose auto/none/utf-8/utf-16/utf-16le/utf-16be text decoding, null_data/--null-data to use NUL as the input line terminator, crlf/--crlf to treat CRLF/CR/LF as line terminators for anchors, text/-a/--text to search binary-extension files as text, no_text/--no-text to disable text mode, word_regexp/-w/--word-regexp for whole-word matches, line_regexp/-x/--line-regexp for whole-line matches, ignore_case/-i/--ignore-case for case-insensitive search, case_sensitive/-s/--case-sensitive to force case-sensitive matching, smart_case/-S/--smart-case for lowercase-only patterns, and invert_match/-v/--invert-match to select non-matching lines. Set no_ignore/--no-ignore to skip .gitignore/.ignore/.rgignore files, no_ignore_dot/--no-ignore-dot to skip .ignore/.rgignore while keeping .gitignore active, no_ignore_vcs/--no-ignore-vcs to skip .gitignore while keeping .ignore/.rgignore active, ignore_file/--ignore-file to add a gitignore-formatted file matched relative to the current working directory, or no_ignore_files/--no-ignore-files to ignore explicit ignore_file inputs; VCS metadata and read-denied paths remain excluded. Set multiline to allow patterns to span lines with dot matching newlines.", nil
1008+
return "Searches text files under path using a regular expression or fixed string. pattern is the canonical search expression; regex/regexp/--regexp/-e are accepted aliases, and pattern_file/--file/-f can read one pattern per line from a file. output_mode may be files, files_with_matches, files_without_matches, content, or count; glob/-g/--glob, iglob/--iglob, type/-t/--type, and type_not/-T/--type-not optionally filter file paths. glob and iglob accept whitespace/comma-separated patterns, negation, and brace alternation; glob_case_insensitive/--glob-case-insensitive makes glob patterns ignore case. content mode supports context, before_context, after_context, -C, -B, -A, -n/--line-number and -N/--no-line-number line-number control, --column column-number output, byte_offset/--byte-offset/-b byte offset output, -H/--with-filename and -I/--no-filename filename prefix control, heading/--heading grouped file headings, path_separator/--path-separator display path separator control, null/--null NUL path terminators/separators, field_match_separator/--field-match-separator and field_context_separator/--field-context-separator output field separators, context_separator/--context-separator and no_context_separator/--no-context-separator context group separator control, offset, head_limit pagination, max_count/-m per-file match limiting, max_columns/--max-columns long-line omission, --max-columns-preview long-line previews, replace/--replace/-r display-only replacement, only_matching/-o/--only-matching matched-text output, vimgrep/--vimgrep per-match line output, passthru/--passthru/--passthrough all-line output, trim/--trim leading-whitespace trimming, stats/--stats aggregate statistics, json/--json NDJSON events, quiet/--quiet/-q output suppression, and hidden/--hidden or no_hidden/--no-hidden hidden file traversal control. Use files/--files to list files that would be searched without requiring pattern, files_with_matches or -l to list files with matches, files_without_match to list files without matches, and count/--count/-c for count mode. Count mode supports count_matches/--count-matches for occurrence counts and include_zero/--include-zero to include zero-count files. Use max_depth/--max-depth/-d to limit directory descent, max_filesize/--max-filesize with optional K/M/G suffix to skip larger files, follow/--follow/-L or no_follow/--no-follow to control symlink traversal, and sort/--sort or sortr/--sortr with path or modified to control result ordering; --sort-files is accepted as a path-sort alias. Use fixed_strings/-F/--fixed-strings for literal matching, encoding/--encoding/-E to choose auto/none/utf-8/utf-16/utf-16le/utf-16be text decoding, null_data/--null-data to use NUL as the input line terminator, crlf/--crlf to treat CRLF/CR/LF as line terminators for anchors, text/-a/--text to disable binary detection and search NUL-containing files as text, binary/--binary to report NUL-containing files that match, no_binary/--no-binary to restore binary filtering, no_text/--no-text to disable text mode, word_regexp/-w/--word-regexp for whole-word matches, line_regexp/-x/--line-regexp for whole-line matches, ignore_case/-i/--ignore-case for case-insensitive search, case_sensitive/-s/--case-sensitive to force case-sensitive matching, smart_case/-S/--smart-case for lowercase-only patterns, and invert_match/-v/--invert-match to select non-matching lines. Set no_ignore/--no-ignore to skip .gitignore/.ignore/.rgignore files, no_ignore_dot/--no-ignore-dot to skip .ignore/.rgignore while keeping .gitignore active, no_ignore_vcs/--no-ignore-vcs to skip .gitignore while keeping .ignore/.rgignore active, ignore_file/--ignore-file to add a gitignore-formatted file matched relative to the current working directory, or no_ignore_files/--no-ignore-files to ignore explicit ignore_file inputs; VCS metadata and read-denied paths remain excluded. Set multiline to allow patterns to span lines with dot matching newlines.", nil
9941009
},
9951010
NormalizeFunc: normalizeGrepRawInput,
9961011
ValidateFunc: validateGrep,
@@ -1264,6 +1279,7 @@ func callGrep(ctx tool.Context, raw json.RawMessage, _ tool.ProgressSink) (contr
12641279
IncludeZero: includeZero,
12651280
ColumnNumbers: grepColumnNumbers(input),
12661281
Text: grepText(input) || nullData,
1282+
Binary: grepBinary(input),
12671283
Encoding: encoding,
12681284
SortMode: sortMode,
12691285
SortReverse: sortReverse,
@@ -1352,6 +1368,8 @@ func callGrep(ctx tool.Context, raw json.RawMessage, _ tool.ProgressSink) (contr
13521368
"fixed_strings": grepFixedStrings(input),
13531369
"text": options.Text,
13541370
"no_text": grepNoText(input),
1371+
"binary": options.Binary,
1372+
"no_binary": grepNoBinary(input),
13551373
"word_regexp": grepWordRegexp(input),
13561374
"line_regexp": grepLineRegexp(input),
13571375
"invert_match": invertMatch,
@@ -1606,19 +1624,45 @@ func collectGrepMatches(root string, displayRoot string, glob string, iglob stri
16061624
if options.HasMaxFilesize && info.Size() > options.MaxFilesize {
16071625
return nil
16081626
}
1609-
if hasBinaryExtension(path) && !options.Text {
1610-
return nil
1611-
}
16121627
if options.Mode == "files" {
16131628
matches = append(matches, grepMatch{Path: displayRel, ModUnix: info.ModTime().UnixNano()})
16141629
return nil
16151630
}
1616-
content, err := readGrepText(path, options.Text, options.Encoding)
1631+
content, err := readGrepText(path, options.Text || options.Binary, options.Encoding)
16171632
if err != nil {
16181633
return nil
16191634
}
16201635
stats.FilesSearched++
16211636
stats.BytesSearched += info.Size()
1637+
if binaryOffset := strings.IndexByte(content, 0); binaryOffset >= 0 && !options.Text {
1638+
if !options.Binary {
1639+
return nil
1640+
}
1641+
if !expr.MatchString(content) {
1642+
if options.Mode == "files_without_matches" {
1643+
matches = append(matches, grepMatch{Path: displayRel, ModUnix: info.ModTime().UnixNano()})
1644+
} else if options.Mode == "count" && options.IncludeZero {
1645+
matches = append(matches, grepMatch{Path: displayRel, Count: 0, ModUnix: info.ModTime().UnixNano()})
1646+
}
1647+
stats.Files[displayRel] = grepFileStats{BytesSearched: info.Size()}
1648+
return nil
1649+
}
1650+
stats.Matches++
1651+
stats.MatchedLines++
1652+
stats.FilesWithMatches++
1653+
stats.Files[displayRel] = grepFileStats{Matches: 1, MatchedLines: 1, BytesSearched: info.Size(), HasMatches: true}
1654+
switch options.Mode {
1655+
case "files_without_matches":
1656+
return nil
1657+
case "files_with_matches":
1658+
matches = append(matches, grepMatch{Path: displayRel, ModUnix: info.ModTime().UnixNano()})
1659+
case "count":
1660+
matches = append(matches, grepMatch{Path: displayRel, Count: 1, ModUnix: info.ModTime().UnixNano()})
1661+
default:
1662+
matches = append(matches, grepMatch{Path: displayRel, Line: 0, Text: fmt.Sprintf(" binary file matches (found \"\\0\" byte around offset %d)", binaryOffset), Matched: true, ModUnix: info.ModTime().UnixNano()})
1663+
}
1664+
return nil
1665+
}
16221666
matchOptions := options
16231667
if options.CountMatches {
16241668
matchOptions.OnlyMatching = true
@@ -2342,13 +2386,13 @@ func formatGrepContentMatch(match grepMatch, options grepOptions) string {
23422386

23432387
func grepContentFields(match grepMatch, options grepOptions) []string {
23442388
fields := make([]string, 0, 4)
2345-
if options.LineNumbers {
2389+
if options.LineNumbers && match.Line > 0 {
23462390
fields = append(fields, strconv.Itoa(match.Line))
23472391
}
23482392
if (options.ColumnNumbers || options.Vimgrep) && match.Matched && match.Column > 0 {
23492393
fields = append(fields, strconv.Itoa(match.Column))
23502394
}
2351-
if options.ByteOffset {
2395+
if options.ByteOffset && match.Line > 0 {
23522396
fields = append(fields, strconv.Itoa(match.ByteOffset))
23532397
}
23542398
fields = append(fields, match.Text)
@@ -3156,6 +3200,20 @@ func grepNoText(input grepInput) bool {
31563200
input.LongNoText
31573201
}
31583202

3203+
func grepBinary(input grepInput) bool {
3204+
if grepNoBinary(input) || grepText(input) {
3205+
return false
3206+
}
3207+
return input.Binary || input.LongBinary
3208+
}
3209+
3210+
func grepNoBinary(input grepInput) bool {
3211+
return input.NoBinary ||
3212+
input.NoBinaryAlt ||
3213+
input.NoBinaryDash ||
3214+
input.LongNoBinary
3215+
}
3216+
31593217
func grepWordRegexp(input grepInput) bool {
31603218
return input.WordRegexp ||
31613219
input.WordRegexpAlt ||

0 commit comments

Comments
 (0)