feat: add line-by-line mode as default, stream without loading files into memory

CHANGELOG.md

@@ -78,7 +78,7 @@ release artifacts on the releases page.
   - Fixes several cross-compilation issues that effected different targets in CI
 - #182 `cargo update` (@CosmicHorrorDev)
   - Bumps dependencies to their latest compatible versions
-- #183 Switch `memmap` -> `memmap2` (@CosmicHorrorDev)
+- #183 Switch file-mapping crate implementation (@CosmicHorrorDev)
   - Switches away from an unmaintained crate
 - #184 Add editor config file matching rustfmt config (@CosmicHorrorDev)
   - Adds an `.editorconfig` file matching the settings listed in the
@@ -121,7 +121,7 @@ release artifacts on the releases page.
 
 ## [0.6.2]
 
-- Fixed pre-allocated memmap buffer size
+- Fixed pre-allocated file-mapping buffer size
 - Fixed failing tests
 
 ## [0.6.0] - 2019-06-15
@@ -188,4 +188,3 @@ To reflect this change, `--input` is also renamed to `--in-place`. This is the f
 
 - Files are now written to [atomically](https://github.com/chmln/sd/issues/3)
 
-

Cargo.toml

@@ -1,48 +1,30 @@
 [workspace]
+resolver = "3"
 members = [
-    ".",
+    "sd",
+    "sd-cli",
     "xtask",
 ]
 
-[workspace.dependencies.clap]
-version = "4.4.6"
-features = ["derive", "deprecated", "wrap_help"]
+[workspace.dependencies]
+tempfile = "3.8.0"
+clap = {version =  "4.4.6", features = ["derive", "wrap_help"]}
+
+
 
 [workspace.package]
 edition = "2024"
 version = "1.0.0"
-
-[package]
-name = "sd"
-version.workspace = true
-edition.workspace = true
-authors = ["Gregory <gregory.mkv@gmail.com>"]
+authors = ["Gregory <gregory.mkv@gmail.com>", "Orión <oriongonza42@pm.me>"]
 description = "An intuitive find & replace CLI"
-readme = "README.md"
+readme = "../README.md"
 keywords = ["sed", "find", "replace", "regex"]
 license = "MIT"
 homepage = "https://github.com/chmln/sd"
 repository = "https://github.com/chmln/sd.git"
 categories = ["command-line-utilities", "text-processing", "development-tools"]
 rust-version = "1.86.0"
 
-[dependencies]
-regex = "1.10.2"
-rayon = "1.8.0"
-memmap2 = "0.9.0"
-tempfile = "3.8.0"
-thiserror = "1.0.50"
-clap.workspace = true
-
-[dev-dependencies]
-assert_cmd = "2.1.1"
-anyhow = "1.0.75"
-clap_mangen = "0.2.14"
-proptest = "1.3.1"
-console = "0.15.7"
-insta = "1.34.0"
-ansi-to-html = "0.1.3"
-regex-automata = "0.4.3"
 
 [profile.release]
 opt-level = 3

README.md

@@ -23,9 +23,11 @@ Simpler syntax for replacing all occurrences:
   - sed: `sed s/before/after/g`
 
 Replace newlines with commas:
-  - sd: `sd '\n' ','`
+  - sd: `sd -A '\n' ','`
   - sed: `sed ':a;N;$!ba;s/\n/,/g'`
 
+  Note: this requires `-A` (across mode) since `\n` is a cross-line pattern.
+
 Extracting stuff out of strings containing slashes:
   - sd: `echo "sample with /path/" | sd '.*(/.*/)' '$1'`
   - sed: `echo "sample with /path/" | sed -E 's/.*(\\/.*\\/)/\1/g'`
@@ -78,6 +80,27 @@ hyperfine --warmup 3 --export-markdown out.md \
 
 Result: ~11.93 times faster
 
+**Line-by-line vs across mode** (1M lines, ~36MB file):
+
+| Command | Mean [ms] | Relative |
+|:---|---:|---:|
+| `sd -A 'foo' 'qux'` (across) | 125.6 ± 14.3 | 1.00 |
+| `sed s/foo/qux/g` | 316.4 ± 30.0 | 2.52 |
+| `sd 'foo' 'qux'` (line-by-line, default) | 357.0 ± 15.0 | 2.84 |
+
+| Command | Mean [ms] | Relative |
+|:---|---:|---:|
+| `sd -A '(\w+) world' '$1 earth'` (across) | 254.0 ± 11.2 | 1.00 |
+| `sd '(\w+) world' '$1 earth'` (line-by-line, default) | 566.7 ± 16.7 | 2.23 |
+| `sed -E 's/(\w+) world/\1 earth/g'` | 4432.7 ± 173.2 | 17.45 |
+
+Line-by-line mode is ~2-3x slower than across mode but still faster than sed for regex replacements. The tradeoff is dramatically lower memory usage:
+
+| Mode | Peak RSS |
+|:---|---:|
+| `sd -A` (across) | 74 MB |
+| `sd` (line-by-line, default) | 3 MB |
+
 ## Installation
 
 Install through
@@ -176,6 +199,21 @@ $ echo "./hello --foo" | sd -- "--foo" "-w"
 ./hello -w
 ```
 
+### Processing modes
+
+By default, sd processes input **line by line**. This means:
+- Low memory usage (only one line in memory at a time)
+- Streaming output for stdin (results appear before EOF)
+- `^` and `$` match the start/end of each line without phantom matches
+- `\s+$` trims trailing whitespace without eating newlines
+
+If you need patterns to match **across line boundaries** (e.g. replacing `\n` or matching multi-line patterns), use the `-A` / `--across` flag:
+
+```sh
+> echo -e "hello\nworld" | sd -A '\n' ','
+hello,world
+```
+
 ### Escaping special characters
 To escape the `$` character, use `$$`:
 

gen/completions/_sd

@@ -23,6 +23,8 @@ _sd() {
 '--preview[Display changes in a human reviewable format (the specifics of the format are likely to change in the future)]' \
 '-F[Treat FIND and REPLACE_WITH args as literal strings]' \
 '--fixed-strings[Treat FIND and REPLACE_WITH args as literal strings]' \
+'-A[Process each input as a whole rather than line by line. This allows patterns to match across line boundaries but uses more memory and prevents streaming]' \
+'--across[Process each input as a whole rather than line by line. This allows patterns to match across line boundaries but uses more memory and prevents streaming]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '-V[Print version]' \

gen/completions/_sd.ps1

@@ -29,6 +29,8 @@ Register-ArgumentCompleter -Native -CommandName 'sd' -ScriptBlock {
             [CompletionResult]::new('--preview', 'preview', [CompletionResultType]::ParameterName, 'Display changes in a human reviewable format (the specifics of the format are likely to change in the future)')
             [CompletionResult]::new('-F', 'F ', [CompletionResultType]::ParameterName, 'Treat FIND and REPLACE_WITH args as literal strings')
             [CompletionResult]::new('--fixed-strings', 'fixed-strings', [CompletionResultType]::ParameterName, 'Treat FIND and REPLACE_WITH args as literal strings')
+            [CompletionResult]::new('-A', 'A ', [CompletionResultType]::ParameterName, 'Process each input as a whole rather than line by line. This allows patterns to match across line boundaries but uses more memory and prevents streaming')
+            [CompletionResult]::new('--across', 'across', [CompletionResultType]::ParameterName, 'Process each input as a whole rather than line by line. This allows patterns to match across line boundaries but uses more memory and prevents streaming')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('-V', 'V ', [CompletionResultType]::ParameterName, 'Print version')

gen/completions/sd.bash

@@ -19,7 +19,7 @@ _sd() {
 
     case "${cmd}" in
         sd)
-            opts="-p -F -n -f -h -V --preview --fixed-strings --max-replacements --flags --help --version <FIND> <REPLACE_WITH> [FILES]..."
+            opts="-p -F -n -f -A -h -V --preview --fixed-strings --max-replacements --flags --across --help --version <FIND> <REPLACE_WITH> [FILES]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0

gen/completions/sd.elv

@@ -26,6 +26,8 @@ set edit:completion:arg-completer[sd] = {|@words|
             cand --preview 'Display changes in a human reviewable format (the specifics of the format are likely to change in the future)'
             cand -F 'Treat FIND and REPLACE_WITH args as literal strings'
             cand --fixed-strings 'Treat FIND and REPLACE_WITH args as literal strings'
+            cand -A 'Process each input as a whole rather than line by line. This allows patterns to match across line boundaries but uses more memory and prevents streaming'
+            cand --across 'Process each input as a whole rather than line by line. This allows patterns to match across line boundaries but uses more memory and prevents streaming'
             cand -h 'Print help (see more with ''--help'')'
             cand --help 'Print help (see more with ''--help'')'
             cand -V 'Print version'

gen/completions/sd.fish

@@ -2,5 +2,6 @@ complete -c sd -s n -l max-replacements -d 'Limit the number of replacements tha
 complete -c sd -s f -l flags -d 'Regex flags. May be combined (like `-f mc`).' -r
 complete -c sd -s p -l preview -d 'Display changes in a human reviewable format (the specifics of the format are likely to change in the future)'
 complete -c sd -s F -l fixed-strings -d 'Treat FIND and REPLACE_WITH args as literal strings'
+complete -c sd -s A -l across -d 'Process each input as a whole rather than line by line. This allows patterns to match across line boundaries but uses more memory and prevents streaming'
 complete -c sd -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c sd -s V -l version -d 'Print version'

gen/sd.1

@@ -8,7 +8,7 @@ sd
 .ie \n(.g .ds Aq \(aq
 .el .ds Aq '
 .SH SYNOPSIS
-\fBsd\fR [\fB\-p\fR|\fB\-\-preview\fR] [\fB\-F\fR|\fB\-\-fixed\-strings\fR] [\fB\-n\fR|\fB\-\-max\-replacements\fR] [\fB\-f\fR|\fB\-\-flags\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] <\fIFIND\fR> <\fIREPLACE_WITH\fR> [\fIFILES\fR] 
+\fBsd\fR [\fB\-p\fR|\fB\-\-preview\fR] [\fB\-F\fR|\fB\-\-fixed\-strings\fR] [\fB\-n\fR|\fB\-\-max\-replacements\fR] [\fB\-f\fR|\fB\-\-flags\fR] [\fB\-A\fR|\fB\-\-across\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] <\fIFIND\fR> <\fIREPLACE_WITH\fR> [\fIFILES\fR] 
 .ie \n(.g .ds Aq \(aq
 .el .ds Aq '
 .SH DESCRIPTION
@@ -40,6 +40,9 @@ s \- make `.` match newlines
 
 w \- match full words only
 .TP
+\fB\-A\fR, \fB\-\-across\fR
+Process each input as a whole rather than line by line. This allows patterns to match across line boundaries but uses more memory and prevents streaming
+.TP
 \fB\-h\fR, \fB\-\-help\fR
 Print help (see a summary with \*(Aq\-h\*(Aq)
 .TP
@@ -82,7 +85,7 @@ lorem ipsum 23
 Indexed capture groups
 \fB$ echo \*(Aqcargo +nightly watch\*(Aq | sd \*(Aq(\\w+)\\s+\\+(\\w+)\\s+(\\w+)\*(Aq \*(Aqcmd: $1, channel: $2, subcmd: $3\*(Aq\fR
 .br
-123 dollars and 45 cents
+cmd: cargo, channel: nightly, subcmd: watch
 .TP
 Find & replace in file
 \fB$ sd \*(Aqwindow.fetch\*(Aq \*(Aqfetch\*(Aq http.js\fR

sd-cli/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "sd-cli"
+version.workspace = true
+edition.workspace = true
+
+[[bin]]
+name = "sd"
+path = "src/main.rs"
+
+[dependencies]
+sd = { path = "../sd" }
+clap.workspace = true
+
+[dev-dependencies]
+assert_cmd = "2.0.12"
+anyhow = "1.0.75"
+clap_mangen = "0.2.14"
+console = "0.15.7"
+regex = "1.10.2"
+insta = "1.34.0"
+ansi-to-html = "0.1.3"
+tempfile.workspace = true

src/cli.rs → sd-cli/src/cli.rs

@@ -57,6 +57,12 @@ w - match full words only
     */
     pub flags: Option<String>,
 
+    #[arg(short = 'A', long = "across")]
+    /// Process each input as a whole rather than line by line. This allows
+    /// patterns to match across line boundaries but uses more memory and
+    /// prevents streaming.
+    pub across: bool,
+
     /// The regexp or string (if using `-F`) to search for.
     pub find: String,
 

sd-cli/src/main.rs

@@ -0,0 +1,41 @@
+mod cli;
+
+use clap::Parser;
+use std::{io::stdout, process};
+
+use sd::{Replacer, Result, Source, process_sources};
+
+fn main() {
+    if let Err(e) = try_main() {
+        eprintln!("error: {e}");
+        process::exit(1);
+    }
+}
+
+fn try_main() -> Result<()> {
+    let options = cli::Options::parse();
+
+    let replacer = Replacer::new(
+        options.find,
+        options.replace_with,
+        options.literal_mode,
+        options.flags,
+        options.replacements,
+    )?;
+
+    let sources = if !options.files.is_empty() {
+        Source::from_paths(options.files)
+    } else {
+        Ok(Source::from_stdin())
+    };
+    let sources = sources?;
+
+    let mut handle = stdout().lock();
+    process_sources(
+        &replacer,
+        &sources,
+        options.preview,
+        !options.across,
+        &mut handle,
+    )
+}