Skip to content

Implement SquashFS-style Option A Compression (LZ4) #326

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Teodor-Tita wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Implement SquashFS-style Option A Compression (LZ4) #326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ documentation = "https://docs.rs/rencfs"
exclude = [".github/"]

[dependencies]
lz4_flex = "0.11"
clap = { version = "4.5.4", features = ["derive", "cargo"] }
libc = "0.2.153"
serde = { version = "1.0.197", features = ["derive"] }
Expand Down
14 changes: 7 additions & 7 deletions benches/crypto_read.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ fn bench_read_1mb_chacha_file(c: &mut Criterion) {
let key = SecretVec::new(Box::new(key));

let file = tempfile::tempfile().unwrap();
let mut writer = crypto::create_write(file, cipher, &key);
let mut writer = crypto::create_write(file, cipher, &key, false);
let mut cursor_random = io::Cursor::new(vec![0; len]);
rand::thread_rng().fill_bytes(cursor_random.get_mut());
cursor_random.seek(io::SeekFrom::Start(0)).unwrap();
Expand All @@ -27,7 +27,7 @@ fn bench_read_1mb_chacha_file(c: &mut Criterion) {
b.iter(|| {
let mut file = file.try_clone().unwrap();
file.seek(io::SeekFrom::Start(0)).unwrap();
let mut reader = crypto::create_read(file, cipher, &key);
let mut reader = crypto::create_read(file, cipher, &key, false);
black_box(&reader);
io::copy(&mut reader, &mut io::sink()).unwrap();
});
Expand All @@ -45,7 +45,7 @@ fn bench_read_1mb_aes_file(c: &mut Criterion) {
c.bench_function("bench_read_1mb_aes_file", |b| {
b.iter(|| {
let file = tempfile::tempfile().unwrap();
let mut writer = crypto::create_write(file, cipher, &key);
let mut writer = crypto::create_write(file, cipher, &key, false);
let mut cursor_random = io::Cursor::new(vec![0; len]);
rand::thread_rng().fill_bytes(cursor_random.get_mut());
cursor_random.seek(io::SeekFrom::Start(0)).unwrap();
Expand All @@ -65,7 +65,7 @@ fn bench_read_1mb_chacha_ram(c: &mut Criterion) {
let key = SecretVec::new(Box::new(key));

let cursor_write = io::Cursor::new(vec![]);
let mut writer = crypto::create_write(cursor_write, cipher, &key);
let mut writer = crypto::create_write(cursor_write, cipher, &key, false);
let mut cursor_random = io::Cursor::new(vec![0; len]);
rand::thread_rng().fill_bytes(cursor_random.get_mut());
cursor_random.seek(io::SeekFrom::Start(0)).unwrap();
Expand All @@ -76,7 +76,7 @@ fn bench_read_1mb_chacha_ram(c: &mut Criterion) {
b.iter(|| {
let mut cursor = cursor_write.clone();
cursor.seek(io::SeekFrom::Start(0)).unwrap();
let mut reader = crypto::create_read(cursor, cipher, &key);
let mut reader = crypto::create_read(cursor, cipher, &key, false);
black_box(&reader);
io::copy(&mut reader, &mut io::sink()).unwrap();
});
Expand All @@ -92,7 +92,7 @@ fn bench_read_1mb_aes_ram(c: &mut Criterion) {
let key = SecretVec::new(Box::new(key));

let cursor_write = io::Cursor::new(vec![]);
let mut writer = crypto::create_write(cursor_write, cipher, &key);
let mut writer = crypto::create_write(cursor_write, cipher, &key, false);
let mut cursor_random = io::Cursor::new(vec![0; len]);
rand::thread_rng().fill_bytes(cursor_random.get_mut());
cursor_random.seek(io::SeekFrom::Start(0)).unwrap();
Expand All @@ -103,7 +103,7 @@ fn bench_read_1mb_aes_ram(c: &mut Criterion) {
b.iter(|| {
let mut cursor = cursor_write.clone();
cursor.seek(io::SeekFrom::Start(0)).unwrap();
let mut reader = crypto::create_read(cursor, cipher, &key);
let mut reader = crypto::create_read(cursor, cipher, &key, false);
black_box(&reader);
io::copy(&mut reader, &mut io::sink()).unwrap();
});
Expand Down
8 changes: 4 additions & 4 deletions examples/crypto_speed.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,9 @@ fn stream_speed(
let path_out2 = Path::new(&path_out).to_path_buf().with_extension("dec");
let _ = fs::remove_file(path_out2.clone());
let mut file_out2 = File::create(path_out2.clone())?;
let mut writer = crypto::create_write(file_out, cipher, key);
let mut writer = crypto::create_write(file_out, cipher, key, true);

@radumarias radumarias Jul 10, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Writer/reader flag mismatch — this example (and crypto_write_read.rs) no longer runs.

Here the file is written with is_compressed=true but read back with false (lines 91/109; same in examples/crypto_write_read.rs: write true at line 35, read false at line 40). The two layouts are mutually unintelligible — different framing (4-byte comp_len header + padding) and different AAD (block_index‖comp_len vs block_index) — so open_within fails on block 0. I reproduced it on this branch: write-true/read-false errors with error opening within.

Beyond fixing these call sites, the mismatch is symptomatic of the design: nothing on disk records whether a stream is compressed, so every reader must guess a positional bool correctly — and the PR's own examples guessed wrong. Consider a Compression enum carried alongside Cipher and persisted in the data-dir format instead of a caller-remembered bool (note crypto::copy_from_file/copy_from_file_exact also hardcode false and can never read a compressed file).

Generated by Claude Code

let size = file_in.metadata()?.len();
let f = || crypto::create_read(File::open(path_out).unwrap(), cipher, key);
let f = || crypto::create_read(File::open(path_out).unwrap(), cipher, key, false);
test_speed(&mut file_in, &mut writer, &mut file_out2, size, f)?;
file_in.seek(io::SeekFrom::Start(0))?;
check_hash(&mut file_in, &mut f())?;
Expand All @@ -101,12 +101,12 @@ fn file_speed(path_in: &str, path_out: &str, cipher: Cipher, key: &SecretVec<u8>
println!("file speed");
let _ = fs::remove_file(path_out);
let mut file_in = File::open(path_in)?;
let mut writer = crypto::create_write(File::create(Path::new(path_out))?, cipher, key);
let mut writer = crypto::create_write(File::create(Path::new(path_out))?, cipher, key, true);
let path_out2 = Path::new(&path_out).to_path_buf().with_extension("dec");
let _ = fs::remove_file(path_out2.clone());
let mut file_out2 = File::create(path_out2.clone())?;
let size = file_in.metadata()?.len();
let f = || crypto::create_read(File::open(path_out).unwrap(), cipher, key);
let f = || crypto::create_read(File::open(path_out).unwrap(), cipher, key, false);
test_speed(&mut file_in, &mut writer, &mut file_out2, size, f)?;
file_in.seek(io::SeekFrom::Start(0)).unwrap();
check_hash(&mut file_in, &mut f())?;
Expand Down
4 changes: 2 additions & 2 deletions examples/crypto_write_read.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@ fn main() -> Result<()> {
}

let mut file = File::open(path_in.clone())?;
let mut writer = crypto::create_write(File::create(out.clone())?, cipher, &key);
let mut writer = crypto::create_write(File::create(out.clone())?, cipher, &key, true);
info!("encrypt file");
io::copy(&mut file, &mut writer).unwrap();
writer.finish()?;

let mut reader = crypto::create_read(File::open(out)?, cipher, &key);
let mut reader = crypto::create_read(File::open(out)?, cipher, &key, false);
info!("read file and compare hash to original one");
let hash1 = crypto::hash_reader(&mut File::open(path_in)?)?;
let hash2 = crypto::hash_reader(&mut reader)?;
Expand Down
24 changes: 10 additions & 14 deletions examples/internal_ring_speed.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,14 +80,12 @@ fn main() -> io::Result<()> {
let len = {
let mut pos = 0;
loop {
match input.read(&mut buffer[pos..]) {
Ok(read) => {
pos += read;
if read == 0 {
break;
}
{
let read = input.read(&mut buffer[pos..])?;
pos += read;
if read == 0 {
break;
}
Err(err) => return Err(err),
}
}
pos
Expand Down Expand Up @@ -139,14 +137,12 @@ fn main() -> io::Result<()> {
let len = {
let mut pos = 0;
loop {
match input.read(&mut buffer[pos..]) {
Ok(read) => {
pos += read;
if read == 0 {
break;
}
{
let read = input.read(&mut buffer[pos..])?;
pos += read;
if read == 0 {
break;
}
Err(err) => return Err(err),
}
}
pos
Expand Down
40 changes: 27 additions & 13 deletions src/crypto.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize};
use shush_rs::{ExposeSecret, SecretString, SecretVec};
use strum_macros::{Display, EnumIter, EnumString};
use thiserror::Error;
use tracing::{debug, error, instrument};
use tracing::{debug, instrument};
use write::CryptoInnerWriter;

use crate::crypto::read::{CryptoRead, CryptoReadSeek, RingCryptoRead};
Expand Down Expand Up @@ -114,89 +114,97 @@ pub fn create_write<W: CryptoInnerWriter + Send + Sync + 'static>(
writer: W,
cipher: Cipher,
key: &SecretVec<u8>,
is_compressed: bool,
) -> impl CryptoWrite<W> {
create_ring_write(writer, cipher, key)
create_ring_write(writer, cipher, key, is_compressed)
}

/// Creates an encrypted writer with seek
pub fn create_write_seek<W: CryptoInnerWriter + Seek + Read + Send + Sync + 'static>(
writer: W,
cipher: Cipher,
key: &SecretVec<u8>,
is_compressed: bool,
) -> impl CryptoWriteSeek<W> {
create_ring_write_seek(writer, cipher, key)
create_ring_write_seek(writer, cipher, key, is_compressed)
}

fn create_ring_write<W: CryptoInnerWriter + Send + Sync>(
writer: W,
cipher: Cipher,
key: &SecretVec<u8>,
is_compressed: bool,
) -> RingCryptoWrite<W> {
let algorithm = match cipher {
Cipher::ChaCha20Poly1305 => &CHACHA20_POLY1305,
Cipher::Aes256Gcm => &AES_256_GCM,
};
RingCryptoWrite::new(writer, false, algorithm, key)
RingCryptoWrite::new(writer, false, algorithm, key, is_compressed)
}

fn create_ring_write_seek<W: CryptoInnerWriter + Seek + Read + Send + Sync>(
writer: W,
cipher: Cipher,
key: &SecretVec<u8>,
is_compressed: bool,
) -> RingCryptoWrite<W> {
let algorithm = match cipher {
Cipher::ChaCha20Poly1305 => &CHACHA20_POLY1305,
Cipher::Aes256Gcm => &AES_256_GCM,
};
RingCryptoWrite::new(writer, true, algorithm, key)
RingCryptoWrite::new(writer, true, algorithm, key, is_compressed)
}

fn create_ring_read<R: Read + Send + Sync>(
reader: R,
cipher: Cipher,
key: &SecretVec<u8>,
is_compressed: bool,
) -> RingCryptoRead<R> {
let algorithm = match cipher {
Cipher::ChaCha20Poly1305 => &CHACHA20_POLY1305,
Cipher::Aes256Gcm => &AES_256_GCM,
};
RingCryptoRead::new(reader, algorithm, key)
RingCryptoRead::new(reader, algorithm, key, is_compressed)
}

fn create_ring_read_seek<R: Read + Seek + Send + Sync>(
reader: R,
cipher: Cipher,
key: &SecretVec<u8>,
is_compressed: bool,
) -> RingCryptoRead<R> {
let algorithm = match cipher {
Cipher::ChaCha20Poly1305 => &CHACHA20_POLY1305,
Cipher::Aes256Gcm => &AES_256_GCM,
};
RingCryptoRead::new_seek(reader, algorithm, key)
RingCryptoRead::new_seek(reader, algorithm, key, is_compressed)
}

/// Creates an encrypted reader
pub fn create_read<R: Read + Send + Sync>(
reader: R,
cipher: Cipher,
key: &SecretVec<u8>,
is_compressed: bool,
) -> impl CryptoRead<R> {
create_ring_read(reader, cipher, key)
create_ring_read(reader, cipher, key, is_compressed)
}

/// Creates an encrypted reader with seek
pub fn create_read_seek<R: Read + Seek + Send + Sync>(
reader: R,
cipher: Cipher,
key: &SecretVec<u8>,
is_compressed: bool,
) -> impl CryptoReadSeek<R> {
create_ring_read_seek(reader, cipher, key)
create_ring_read_seek(reader, cipher, key, is_compressed)
}

#[allow(clippy::missing_errors_doc)]
pub fn encrypt(s: &SecretString, cipher: Cipher, key: &SecretVec<u8>) -> Result<String> {
let mut cursor = io::Cursor::new(vec![]);
let mut writer = create_write(cursor, cipher, key);
let mut writer = create_write(cursor, cipher, key, false);
writer.write_all(s.expose_secret().as_bytes())?;
cursor = writer.finish()?;
let v = cursor.into_inner();
Expand All @@ -209,7 +217,7 @@ pub fn decrypt(s: &str, cipher: Cipher, key: &SecretVec<u8>) -> Result<SecretStr
let vec = BASE64.decode(s)?;
let cursor = io::Cursor::new(vec);

let mut reader = create_read(cursor, cipher, key);
let mut reader = create_read(cursor, cipher, key, false);
let mut decrypted = String::new();
reader.read_to_string(&mut decrypted)?;
Ok(SecretString::new(Box::new(decrypted)))
Expand Down Expand Up @@ -323,7 +331,12 @@ pub fn copy_from_file(
return Ok(0);
}
// create a new reader by reading from the beginning of the file
let mut reader = create_read(OpenOptions::new().read(true).open(file)?, cipher, key);
let mut reader = create_read(
OpenOptions::new().read(true).open(file)?,
cipher,
key,
false,
);
// move read position to the write position
let pos2 = stream_util::seek_forward(&mut reader, pos, stop_on_eof)?;
if pos2 < pos {
Expand Down Expand Up @@ -357,7 +370,7 @@ where
W: CryptoInnerWriter + Send + Sync + 'static,
T: serde::Serialize + ?Sized,
{
let mut writer = create_write(writer, cipher, key);
let mut writer = create_write(writer, cipher, key, false);
bincode::serialize_into(&mut writer, value)?;
let writer = writer.finish()?;
Ok(writer)
Expand Down Expand Up @@ -415,6 +428,7 @@ mod tests {
File::create(encrypted_file_path.clone()).unwrap(),
cipher,
key,
false,
);
io::copy(&mut file, &mut writer).unwrap();
writer.finish().unwrap();
Expand Down
Loading