From d08170596877e3a05444369d8eca4f206324009f Mon Sep 17 00:00:00 2001 From: DanieCuevas <43822444+DanielCuevas1208@users.noreply.github.com> Date: Sun, 2 Aug 2026 19:46:19 -0700 Subject: [PATCH] feat: extend leaflitter --- .github/workflows/ci.yml | 77 ++++ .gitignore | 2 + CMakeLists.txt | 106 ++++++ LICENSE | 21 ++ README.md | 110 ++++++ cmake/leaflitterConfig.cmake.in | 5 + docs/FORMAT.md | 207 +++++++++++ docs/ROADMAP.md | 53 +++ samples/CMakeLists.txt | 2 + samples/roundtrip.c | 61 ++++ src/lf_adler32.c | 26 ++ src/lf_adler32.h | 9 + src/lf_bitio.c | 25 +- src/lf_bitio.h | 7 + src/lf_block.c | 621 ++++++++++++++++++++++++++++++++ src/lf_block.h | 32 ++ src/lf_buf.c | 9 + src/lf_buf.h | 2 + src/lf_common.c | 43 +++ src/lf_common.h | 4 +- src/lf_match.c | 144 ++++++++ src/lf_match.h | 41 +++ src/lf_stream.c | 251 +++++++++++++ src/lf_stream.h | 25 ++ tests/CMakeLists.txt | 14 + tests/test_adler32.c | 92 +++++ tests/test_bitio.c | 178 +++++++++ tests/test_block.c | 180 +++++++++ tests/test_buf.c | 135 +++++++ tests/test_huffman.c | 241 +++++++++++++ tests/test_match.c | 132 +++++++ tests/test_stream.c | 486 +++++++++++++++++++++++++ tools/CMakeLists.txt | 3 + tools/leaflitter.c | 202 +++++++++++ 34 files changed, 3543 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 CMakeLists.txt create mode 100644 LICENSE create mode 100644 README.md create mode 100644 cmake/leaflitterConfig.cmake.in create mode 100644 docs/FORMAT.md create mode 100644 docs/ROADMAP.md create mode 100644 samples/CMakeLists.txt create mode 100644 samples/roundtrip.c create mode 100644 src/lf_adler32.c create mode 100644 src/lf_adler32.h create mode 100644 src/lf_block.c create mode 100644 src/lf_block.h create mode 100644 src/lf_common.c create mode 100644 src/lf_match.c create mode 100644 src/lf_match.h create mode 100644 src/lf_stream.c create mode 100644 src/lf_stream.h create mode 100644 tests/CMakeLists.txt create mode 100644 tests/test_adler32.c create mode 100644 tests/test_bitio.c create mode 100644 tests/test_block.c create mode 100644 tests/test_buf.c create mode 100644 tests/test_huffman.c create mode 100644 tests/test_match.c create mode 100644 tests/test_stream.c create mode 100644 tools/CMakeLists.txt create mode 100644 tools/leaflitter.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d7f72ae --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,77 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + build: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: linux-gcc + os: ubuntu-latest + compiler: gcc + flags: "" + - name: linux-clang + os: ubuntu-latest + compiler: clang + flags: "" + - name: macos-clang + os: macos-latest + compiler: clang + flags: "" + - name: windows-msvc + os: windows-latest + compiler: "" + flags: "" + + steps: + - uses: actions/checkout@v4 + + - name: Configure + run: | + if [ "${{ matrix.compiler }}" != "" ]; then + export CC=${{ matrix.compiler }} + fi + cmake -B build -DCMAKE_BUILD_TYPE=Release -DLEAFLITTER_WERROR=ON ${{ matrix.flags }} + shell: bash + + - name: Build + run: cmake --build build --config Release + + - name: Test + run: ctest --test-dir build -C Release --output-on-failure + + - name: Install + run: | + cmake --install build --config Release --prefix install-dir + test -f install-dir/include/lf_stream.h + test -f install-dir/lib/libleaflitter.a -o -f install-dir/lib/leaflitter.lib + shell: bash + + sanitize: + name: sanitize + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Configure + run: > + cmake -B build + -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_C_COMPILER=clang + -DLEAFLITTER_WERROR=ON + -DCMAKE_C_FLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer" + + - name: Build + run: cmake --build build + + - name: Test + run: ctest --test-dir build --output-on-failure + env: + ASAN_OPTIONS: detect_leaks=1 diff --git a/.gitignore b/.gitignore index 9371e03..89f441c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ build/ build-*/ out/ cmake-build-*/ +install*/ +CMakeUserPresets.json # Compiled objects and libraries *.o diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..68c544d --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,106 @@ +cmake_minimum_required(VERSION 3.16) + +project(leaflitter + VERSION 1.1.0 + DESCRIPTION "A lossless byte-stream compressor that uses LZ77 matching and canonical Huffman coding" + LANGUAGES C) + +include(CTest) + +option(LEAFLITTER_BUILD_TESTS "Build the unit tests" ON) +option(LEAFLITTER_BUILD_TOOLS "Build the command line tools" ON) +option(LEAFLITTER_BUILD_SHARED "Build a shared library in addition to the static library" OFF) +option(LEAFLITTER_WERROR "Treat compiler warnings as errors" OFF) + +set(LEAFLITTER_SOURCES + src/lf_adler32.c + src/lf_bitio.c + src/lf_block.c + src/lf_buf.c + src/lf_common.c + src/lf_huffman.c + src/lf_match.c + src/lf_stream.c) + +add_library(leaflitter STATIC ${LEAFLITTER_SOURCES}) +add_library(leaflitter::leaflitter ALIAS leaflitter) + +target_include_directories(leaflitter + PUBLIC + $ + $) + +target_compile_features(leaflitter PUBLIC c_std_11) + +if(LEAFLITTER_BUILD_SHARED) + add_library(leaflitter_shared SHARED ${LEAFLITTER_SOURCES}) + target_include_directories(leaflitter_shared + PUBLIC + $ + $) + target_compile_features(leaflitter_shared PUBLIC c_std_11) +endif() + +if(MSVC) + target_compile_options(leaflitter PRIVATE /W4) + if(LEAFLITTER_WERROR) + target_compile_options(leaflitter PRIVATE /WX) + endif() + if(LEAFLITTER_BUILD_SHARED) + target_compile_options(leaflitter_shared PRIVATE /W4) + endif() +else() + target_compile_options(leaflitter PRIVATE -Wall -Wextra -Wpedantic) + if(LEAFLITTER_WERROR) + target_compile_options(leaflitter PRIVATE -Werror) + endif() + if(LEAFLITTER_BUILD_SHARED) + target_compile_options(leaflitter_shared PRIVATE -Wall -Wextra -Wpedantic) + endif() +endif() + +if(LEAFLITTER_BUILD_TESTS AND BUILD_TESTING) + add_subdirectory(tests) +endif() + +if(LEAFLITTER_BUILD_TOOLS) + add_subdirectory(tools) + add_subdirectory(samples) +endif() + +include(CMakePackageConfigHelpers) +include(GNUInstallDirs) + +install(TARGETS leaflitter + EXPORT leaflitterTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +if(LEAFLITTER_BUILD_SHARED) + install(TARGETS leaflitter_shared + EXPORT leaflitterTargets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() + +install(DIRECTORY src/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + FILES_MATCHING PATTERN "*.h") + +install(EXPORT leaflitterTargets + FILE leaflitterTargets.cmake + NAMESPACE leaflitter:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/leaflitter) + +configure_package_config_file( + cmake/leaflitterConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/leaflitterConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/leaflitter) + +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/leaflitterConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion) + +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/leaflitterConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/leaflitterConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/leaflitter) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2e01917 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 leaflitter contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..8e4c6d5 --- /dev/null +++ b/README.md @@ -0,0 +1,110 @@ +# leaflitter + +leaflitter compresses byte streams with LZ77 matching and canonical Huffman +coding. The design follows DEFLATE but stays simpler. The library is small, +portable, and free of external dependencies. + +## Features + +- LZ77 hash-chain match finder with a sliding window +- Canonical Huffman coding with a 15-bit length limit +- Stored and Huffman block types +- Adler-32 integrity check on decompression +- C11 API and a command line tool +- Deterministic output for a given input +- No third-party runtime dependencies + +## Build + +leaflitter uses CMake. Configure, build, and test from the project root. + +``` +cmake -B build +cmake --build build +ctest --test-dir build +``` + +Install the library and headers with the following command. + +``` +cmake --install build +``` + +The tests use the vendored Unity framework. Set `BUILD_TESTING=OFF` to skip +the tests. Set `LEAFLITTER_BUILD_TOOLS=OFF` to skip the tool and the sample. + +## Use the library + +Compress and decompress a byte buffer in memory. + +```c +#include "lf_buf.h" +#include "lf_stream.h" + +uint8_t input[] = "hello hello hello"; +lf_buf packed; +lf_buf plain; +lf_result r; + +lf_buf_init(&packed); +lf_buf_init(&plain); + +r = lf_compress(NULL, input, sizeof(input) - 1, &packed); +if (r != LF_OK) { + return r; +} + +r = lf_decompress(packed.data, packed.len, &plain); +if (r != LF_OK) { + return r; +} +``` + +Pass `NULL` for options to use the defaults. Build a custom `lf_options` +struct to change the block size, window size, or match chain length. Call +`lf_options_default` to fill that struct first. + +Compression accepts inputs up to 4 GiB. Decompression accepts any valid +stream. Use `lf_compress_bound` to size an output buffer before compression. + +## Use the command line tool + +The tool compresses and decompresses files. + +``` +leaflitter c +leaflitter d +leaflitter i +``` + +Command `c` compresses `` to ``. Command `d` restores the original +bytes. Command `i` prints the stream header. + +## Format + +The container format is documented in `docs/FORMAT.md`. The header stores the +original size, an Adler-32 checksum, and the window size. A stream ends with +a final block flag. + +The format is new. It is not compatible with DEFLATE or gzip. + +## Roadmap + +See `docs/ROADMAP.md` for the completed work and the planned work. + +## Limitations + +- Compression inputs are limited to 4 GiB. +- The code-length tree is fixed, not adaptive. +- The encoder searches one hash chain per position. +- Compression is single-threaded. + +## Testing + +The tests use Unity and run through CTest. Every module has a test file. +Round-trip tests cover small, large, and random inputs. Golden vectors lock +the byte format. Corruption tests check that bad streams fail cleanly. + +## License + +This project is licensed under the MIT License. See `LICENSE` for details. diff --git a/cmake/leaflitterConfig.cmake.in b/cmake/leaflitterConfig.cmake.in new file mode 100644 index 0000000..72e9110 --- /dev/null +++ b/cmake/leaflitterConfig.cmake.in @@ -0,0 +1,5 @@ +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/leaflitterTargets.cmake") + +check_required_components(leaflitter) diff --git a/docs/FORMAT.md b/docs/FORMAT.md new file mode 100644 index 0000000..9b9f9ea --- /dev/null +++ b/docs/FORMAT.md @@ -0,0 +1,207 @@ +# leaflitter format + +This document describes the leaflitter byte format, version 1. It is a +reference for implementers. Bits are read most-significant-bit first. + +## Container + +A compressed stream starts with a 24-byte header. The header records the +original size, an Adler-32 checksum, and the window size. + +| Offset | Size | Field | +| ------ | ---- | ----- | +| 0 | 1 | Magic byte `0x4C` ('L') | +| 1 | 1 | Magic byte `0x46` ('F') | +| 2 | 1 | Magic byte `0x01` | +| 3 | 1 | Format version, currently 1 | +| 4 | 8 | Original size, little-endian | +| 12 | 4 | Adler-32 of the original data | +| 16 | 4 | Window size in bytes | +| 20 | 4 | Reserved, must be zero | +| 24 | ... | Blocks | + +The header is followed by one or more blocks. Each block is self-contained. +A final block flag marks the end of the stream. + +The checksum covers the original bytes. The decoder verifies the checksum +after decoding. The decoder rejects streams with a wrong checksum. + +## Block header + +Every block starts with one header byte. + +| Bit | Meaning | +| --- | ------- | +| 0-1 | Block type | +| 2 | Final block flag | +| 3-7 | Reserved, must be zero | + +Block type 0 is a stored block. Block type 1 is a Huffman block. Types 2 and +3 are reserved. + +## Stored block + +A stored block copies its bytes without compression. + +| Field | Size | Notes | +| ----- | ---- | ----- | +| Header | 1 | Type 0 | +| Length | 4 | Byte count, little-endian | +| Data | Length | Raw bytes | + +The length must not be zero. + +## Huffman block + +A Huffman block carries code lengths and a token stream. + +| Field | Size | Notes | +| ----- | ---- | ----- | +| Header | 1 | Type 1 | +| nlit | 2 | Literal and length code lengths | +| ndist | 2 | Distance code lengths | +| Literal lengths | varies | Encoded code lengths | +| Distance lengths | varies | Encoded code lengths | +| Tokens | varies | Huffman-coded tokens | +| Padding | varies | Zero bits to the byte boundary | + +`nlit` must be between 257 and 288. `ndist` must be between 1 and 30. + +## Symbol alphabets + +Literal and length symbols cover three roles. + +- Symbols 0 to 255 are literal bytes. +- Symbol 256 marks the end of a block. +- Symbols 257 to 272 are length codes. + +Symbols 273 to 285 are reserved. A decoder must reject them. + +Distance symbols 0 to 29 encode distances from 1 to 65534. + +## Length codes + +Length codes cover lengths 3 to 258. Each code has a base and extra bits. + +| Code | Base | Extra bits | +| ---- | ---- | ---------- | +| 257 | 3 | 0 | +| 258 | 4 | 0 | +| 259 | 5 | 0 | +| 260 | 6 | 0 | +| 261 | 7 | 1 | +| 262 | 9 | 1 | +| 263 | 11 | 2 | +| 264 | 15 | 2 | +| 265 | 19 | 3 | +| 266 | 27 | 3 | +| 267 | 35 | 4 | +| 268 | 51 | 4 | +| 269 | 67 | 5 | +| 270 | 99 | 5 | +| 271 | 131 | 6 | +| 272 | 195 | 6 | + +The length is the base plus the extra bits. A length code reads its extra +bits right after its Huffman code. + +## Distance codes + +Distance codes cover distances 1 to 65534. + +| Code | Base | Extra bits | +| ---- | ---- | ---------- | +| 0 | 1 | 0 | +| 1 | 2 | 0 | +| 2 | 3 | 1 | +| 3 | 5 | 1 | +| 4 | 7 | 2 | +| 5 | 11 | 2 | +| 6 | 15 | 3 | +| 7 | 23 | 3 | +| 8 | 31 | 4 | +| 9 | 47 | 4 | +| 10 | 63 | 5 | +| 11 | 95 | 5 | +| 12 | 127 | 6 | +| 13 | 191 | 6 | +| 14 | 255 | 7 | +| 15 | 383 | 7 | +| 16 | 511 | 8 | +| 17 | 767 | 8 | +| 18 | 1023 | 9 | +| 19 | 1535 | 9 | +| 20 | 2047 | 10 | +| 21 | 3071 | 10 | +| 22 | 4095 | 11 | +| 23 | 6143 | 11 | +| 24 | 8191 | 12 | +| 25 | 12287 | 12 | +| 26 | 16383 | 13 | +| 27 | 24575 | 13 | +| 28 | 32767 | 14 | +| 29 | 49151 | 14 | + +The distance is the base plus the extra bits. A decoder must reject a +distance that exceeds the window size. + +## Code-length coding + +The code lengths use a run-length scheme. A fixed code-length tree encodes +the symbols. The fixed tree has 19 symbols. + +| Symbol | Meaning | Extra bits | +| ------ | ------- | ---------- | +| 0-15 | Literal length value | 0 | +| 16 | Repeat the last length | 2 | +| 17 | Emit 3 to 10 zeros | 3 | +| 18 | Emit 11 to 138 zeros | 7 | + +Symbol 16 repeats the last length. Symbol 17 emits a run of zeros. Symbol 18 +emits a longer run of zeros. The extra bits select the run length within the +range. + +The fixed tree uses these code lengths for symbols 0 to 18. + +``` +4 5 5 5 5 6 6 6 6 6 6 6 6 6 6 6 3 2 2 +``` + +The encoder writes exactly `nlit` literal lengths. It then writes exactly +`ndist` distance lengths. The decoder must reject an encoding that produces +more values than requested. + +## Token stream + +The token stream follows the code lengths. Every token is one of two kinds. + +A literal token is one Huffman symbol in the literal and length tree. The +symbol value is the literal byte. + +A match token has four parts. + +1. The length symbol in the literal and length tree +2. The length extra bits +3. The distance symbol in the distance tree +4. The distance extra bits + +A match copies `length` bytes from `distance` bytes back. The decoder copies +byte by byte. That order allows overlapping matches. + +The end-of-block symbol ends the token stream. The block then pads with zero +bits to the byte boundary. + +## Decoder rules + +The decoder must reject malformed streams. + +- The magic bytes must match. +- The format version must be 1. +- The reserved fields must be zero. +- `nlit` and `ndist` must stay in range. +- Reserved symbols are invalid. +- A distance must not exceed the produced output. +- A block must not exceed the declared size. + +A decoder returns a specific error for each failure class. See `lf_result` +in `lf_common.h`. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md new file mode 100644 index 0000000..f1908e3 --- /dev/null +++ b/docs/ROADMAP.md @@ -0,0 +1,53 @@ +# leaflitter roadmap + +This page tracks the project plan. Each item has a status. + +## Complete + +### Release 1.1.0 - full pipeline + +This release turns the building blocks into a working compressor. + +- Byte buffer (`lf_buf`) +- Bit reader and writer (`lf_bitio`) +- Canonical Huffman coder (`lf_huffman`) +- Adler-32 checksum (`lf_adler32`) +- LZ77 hash-chain match finder (`lf_match`) +- Block codec with stored and Huffman blocks (`lf_block`) +- Stream container with header and checksum (`lf_stream`) +- Command line tool +- CMake build system +- Unit tests with Unity +- GitHub Actions workflow + +### Release 1.0.0 - foundations + +- Byte buffer (`lf_buf`) +- Bit reader and writer (`lf_bitio`) +- Canonical Huffman coder (`lf_huffman`) + +## Planned + +The list below is not committed. Items appear in rough priority order. + +### Improve compression + +- [ ] Adaptive code-length tree +- [ ] Lazy matching in the encoder +- [ ] Better hash functions and chain selection +- [ ] Skip-huffman fast path for incompressible data +- [ ] Match-finder tuning per data type + +### Expand the API + +- [ ] Incremental compression API +- [ ] Incremental decompression API +- [ ] Preset dictionary support +- [ ] Random access to blocks + +### Expand the tooling + +- [ ] Level presets in the command line tool +- [ ] Parallel block compression +- [ ] Benchmark harness against DEFLATE tools +- [ ] Bindings for other languages diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt new file mode 100644 index 0000000..4d572ca --- /dev/null +++ b/samples/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(leaflitter_sample roundtrip.c) +target_link_libraries(leaflitter_sample PRIVATE leaflitter::leaflitter) diff --git a/samples/roundtrip.c b/samples/roundtrip.c new file mode 100644 index 0000000..aaec159 --- /dev/null +++ b/samples/roundtrip.c @@ -0,0 +1,61 @@ +#include "lf_buf.h" +#include "lf_common.h" +#include "lf_stream.h" + +#include +#include + +static int check_roundtrip(const char *text) +{ + lf_buf packed; + lf_buf plain; + lf_result r; + size_t n = strlen(text); + int ok = 0; + + lf_buf_init(&packed); + lf_buf_init(&plain); + + r = lf_compress(NULL, text, n, &packed); + if (r != LF_OK) { + fprintf(stderr, "compress: %s\n", lf_result_name(r)); + goto done; + } + r = lf_decompress(packed.data, packed.len, &plain); + if (r != LF_OK) { + fprintf(stderr, "decompress: %s\n", lf_result_name(r)); + goto done; + } + + ok = (plain.len == n) && (n == 0 || memcmp(plain.data, text, n) == 0); + if (ok) { + printf("%4zu -> %4zu bytes\n", n, packed.len); + } else { + fprintf(stderr, "roundtrip mismatch\n"); + } + +done: + lf_buf_free(&packed); + lf_buf_free(&plain); + return ok ? 0 : 1; +} + +int main(void) +{ + const char *text = + "leaflitter compresses bytes with LZ77 matches and Huffman codes. " + "leaflitter compresses bytes with LZ77 matches and Huffman codes. " + "leaflitter compresses bytes with LZ77 matches and Huffman codes."; + + printf("leaflitter %s\n", lf_version_string()); + if (check_roundtrip(text) != 0) { + return 1; + } + if (check_roundtrip("x") != 0) { + return 1; + } + if (check_roundtrip("") != 0) { + return 1; + } + return 0; +} diff --git a/src/lf_adler32.c b/src/lf_adler32.c new file mode 100644 index 0000000..2ef593e --- /dev/null +++ b/src/lf_adler32.c @@ -0,0 +1,26 @@ +#include "lf_adler32.h" + +#define LF_ADLER_BASE 65521u +#define LF_ADLER_NMAX 5552u + +uint32_t lf_adler32(const void *data, size_t len) +{ + const uint8_t *p = (const uint8_t *)data; + uint32_t a = 1; + uint32_t b = 0; + + while (len > 0) { + size_t chunk = len < LF_ADLER_NMAX ? len : LF_ADLER_NMAX; + size_t i; + + len -= chunk; + for (i = 0; i < chunk; i++) { + a += p[i]; + b += a; + } + p += chunk; + a %= LF_ADLER_BASE; + b %= LF_ADLER_BASE; + } + return (b << 16) | a; +} diff --git a/src/lf_adler32.h b/src/lf_adler32.h new file mode 100644 index 0000000..b32425b --- /dev/null +++ b/src/lf_adler32.h @@ -0,0 +1,9 @@ +#ifndef LF_ADLER32_H +#define LF_ADLER32_H + +#include "lf_common.h" + +/* Computes the Adler-32 checksum of `data`. An empty input yields 1. */ +LF_EXPORT uint32_t lf_adler32(const void *data, size_t len); + +#endif /* LF_ADLER32_H */ diff --git a/src/lf_bitio.c b/src/lf_bitio.c index 124f3f3..d51ecd9 100644 --- a/src/lf_bitio.c +++ b/src/lf_bitio.c @@ -29,7 +29,7 @@ lf_result lf_bwriter_put(lf_bwriter *w, uint32_t value, unsigned nbits) return r; } w->nbits -= 8u; - w->acc &= 0x7Fu; + w->acc &= ((uint64_t)1 << w->nbits) - 1u; } return LF_OK; } @@ -94,3 +94,26 @@ size_t lf_breader_bytes_read(lf_breader *r) { return r->pos; } + +void lf_breader_align(lf_breader *r) +{ + r->nbits = 0; + r->acc = 0; +} + +int lf_breader_read_bytes(lf_breader *r, uint8_t *dst, size_t n) +{ + if (r->nbits != 0) { + r->valid = 0; + return 0; + } + if (r->pos > r->len || n > r->len - r->pos) { + r->valid = 0; + return 0; + } + if (n > 0) { + memcpy(dst, r->data + r->pos, n); + r->pos += n; + } + return 1; +} diff --git a/src/lf_bitio.h b/src/lf_bitio.h index c1b77c8..5b693a1 100644 --- a/src/lf_bitio.h +++ b/src/lf_bitio.h @@ -54,4 +54,11 @@ LF_INLINE int lf_breader_finished(lf_breader *r) LF_EXPORT size_t lf_breader_bytes_read(lf_breader *r); +/* Drops any partial byte still buffered. Call only at a byte boundary. */ +LF_EXPORT void lf_breader_align(lf_breader *r); + +/* Copies `n` whole bytes into `dst`. Returns 1 on success, 0 on truncation. + Call only at a byte boundary. */ +LF_EXPORT int lf_breader_read_bytes(lf_breader *r, uint8_t *dst, size_t n); + #endif /* LF_BITIO_H */ diff --git a/src/lf_block.c b/src/lf_block.c new file mode 100644 index 0000000..9faf737 --- /dev/null +++ b/src/lf_block.c @@ -0,0 +1,621 @@ +#include "lf_block.h" + +#include "lf_huffman.h" + +#include +#include + +/* + * Length codes 257..272 cover lengths 3..258. + * Distance codes 0..29 cover distances 1..65534. + */ + +#define LF_LEN_CODES 16u +#define LF_DIST_CODES 30u +#define LF_FIRST_LEN 257u +#define LF_CL_SYMBOLS 19u + +#define LF_BLOCK_TYPE_STORED 0u +#define LF_BLOCK_TYPE_HUFFMAN 1u +#define LF_BLOCK_FLAG_FINAL 0x04u + +static const uint8_t lf_len_extra[LF_LEN_CODES] = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6 +}; +static const uint16_t lf_len_base[LF_LEN_CODES] = { + 3, 4, 5, 6, 7, 9, 11, 15, 19, 27, 35, 51, 67, 99, 131, 195 +}; +static const uint8_t lf_dist_extra[LF_DIST_CODES] = { + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, + 10, 10, 11, 11, 12, 12, 13, 13, 14, 14 +}; +static const uint16_t lf_dist_base[LF_DIST_CODES] = { + 1, 2, 3, 5, 7, 11, 15, 23, 31, 47, 63, 95, 127, 191, 255, 383, + 511, 767, 1023, 1535, 2047, 3071, 4095, 6143, 8191, 12287, 16383, + 24575, 32767, 49151 +}; + +/* Fixed canonical code-length tree. Symbols 16, 17, 18 are run codes. */ +static const uint8_t lf_cl_len[LF_CL_SYMBOLS] = { + 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 3, 2, 2 +}; + +typedef struct lf_token { + uint16_t kind; + uint16_t lit; + uint16_t len_code; + uint16_t len_extra; + uint16_t dist_code; + uint16_t dist_extra; +} lf_token; + +static unsigned lf_len_lookup(unsigned len, unsigned *extra) +{ + unsigned k; + + for (k = 0; k < LF_LEN_CODES; k++) { + unsigned top = (unsigned)lf_len_base[k] + ((1u << lf_len_extra[k]) - 1u); + + if (len <= top) { + *extra = len - (unsigned)lf_len_base[k]; + return k; + } + } + *extra = 0; + return 0; +} + +static unsigned lf_dist_lookup(uint32_t dist, unsigned *extra) +{ + unsigned k; + + for (k = 0; k < LF_DIST_CODES; k++) { + uint32_t top = (uint32_t)lf_dist_base[k] + + ((1u << lf_dist_extra[k]) - 1u); + + if (dist <= top) { + *extra = dist - (uint32_t)lf_dist_base[k]; + return k; + } + } + *extra = 0; + return 0; +} + +static void lf_cl_build(uint32_t *codes, uint8_t *lengths) +{ + memcpy(lengths, lf_cl_len, sizeof(lf_cl_len)); + lf_huff_canonical(codes, lf_cl_len, LF_CL_SYMBOLS); +} + +static lf_result lf_cl_write(lf_bwriter *w, const uint8_t *lengths, size_t n, + const uint32_t *cl_code, const uint8_t *cl_len) +{ + size_t i = 0; + lf_result r; + + while (i < n) { + uint8_t v = lengths[i]; + size_t run = 1; + + while (i + run < n && lengths[i + run] == v) { + run++; + } + + if (v == 0) { + while (run > 0) { + if (run >= 11) { + size_t take = run > 138 ? 138 : run; + + r = lf_bwriter_put_code(w, cl_code[18], cl_len[18]); + if (r != LF_OK) { + return r; + } + r = lf_bwriter_put(w, (uint32_t)(take - 11), 7); + if (r != LF_OK) { + return r; + } + i += take; + run -= take; + } else if (run >= 3) { + r = lf_bwriter_put_code(w, cl_code[17], cl_len[17]); + if (r != LF_OK) { + return r; + } + r = lf_bwriter_put(w, (uint32_t)(run - 3), 3); + if (r != LF_OK) { + return r; + } + i += run; + run = 0; + } else if (run == 2) { + r = lf_bwriter_put_code(w, cl_code[0], cl_len[0]); + if (r != LF_OK) { + return r; + } + r = lf_bwriter_put_code(w, cl_code[0], cl_len[0]); + if (r != LF_OK) { + return r; + } + i += 2; + run = 0; + } else { + r = lf_bwriter_put_code(w, cl_code[0], cl_len[0]); + if (r != LF_OK) { + return r; + } + i += 1; + run = 0; + } + } + } else { + r = lf_bwriter_put_code(w, cl_code[v], cl_len[v]); + if (r != LF_OK) { + return r; + } + i++; + run--; + while (run > 0) { + if (run >= 3) { + size_t take = run > 6 ? 6 : run; + + r = lf_bwriter_put_code(w, cl_code[16], cl_len[16]); + if (r != LF_OK) { + return r; + } + r = lf_bwriter_put(w, (uint32_t)(take - 3), 2); + if (r != LF_OK) { + return r; + } + i += take; + run -= take; + } else { + r = lf_bwriter_put_code(w, cl_code[v], cl_len[v]); + if (r != LF_OK) { + return r; + } + i++; + run--; + } + } + } + } + return LF_OK; +} + +static lf_result lf_cl_read(lf_breader *r, uint8_t *lengths, size_t n) +{ + lf_huff_dec cl; + uint8_t cl_len[LF_CL_SYMBOLS]; + uint32_t cl_code[LF_CL_SYMBOLS]; + size_t i = 0; + int last = 0; + + lf_cl_build(cl_code, cl_len); + lf_huff_dec_init(&cl, cl_len, LF_CL_SYMBOLS); + + while (i < n) { + int sym = lf_huff_dec_symbol(&cl, r); + uint32_t extra; + size_t ncopy; + + if (sym < 0) { + return LF_ERR_TRUNCATED; + } + if (sym <= 15) { + lengths[i++] = (uint8_t)sym; + last = sym; + } else if (sym == 16) { + extra = lf_breader_get(r, 2); + if (!lf_breader_ok(r)) { + return LF_ERR_TRUNCATED; + } + ncopy = 3u + extra; + if (i + ncopy > n) { + return LF_ERR_INVALID_DATA; + } + while (ncopy-- > 0) { + lengths[i++] = (uint8_t)last; + } + } else if (sym == 17) { + extra = lf_breader_get(r, 3); + if (!lf_breader_ok(r)) { + return LF_ERR_TRUNCATED; + } + ncopy = 3u + extra; + if (i + ncopy > n) { + return LF_ERR_INVALID_DATA; + } + while (ncopy-- > 0) { + lengths[i++] = 0; + } + last = 0; + } else { + extra = lf_breader_get(r, 7); + if (!lf_breader_ok(r)) { + return LF_ERR_TRUNCATED; + } + ncopy = 11u + extra; + if (i + ncopy > n) { + return LF_ERR_INVALID_DATA; + } + while (ncopy-- > 0) { + lengths[i++] = 0; + } + last = 0; + } + } + return LF_OK; +} + +static lf_result lf_write_tokens(lf_bwriter *w, const lf_token *tokens, + size_t ntok, const uint32_t *lit_code, + const uint8_t *lit_len, const uint32_t *dist_code, + const uint8_t *dist_len) +{ + size_t i; + lf_result r; + + for (i = 0; i < ntok; i++) { + const lf_token *t = &tokens[i]; + + if (t->kind == 0) { + r = lf_bwriter_put_code(w, lit_code[t->lit], lit_len[t->lit]); + if (r != LF_OK) { + return r; + } + } else { + uint32_t sym = LF_FIRST_LEN + t->len_code; + + r = lf_bwriter_put_code(w, lit_code[sym], lit_len[sym]); + if (r != LF_OK) { + return r; + } + r = lf_bwriter_put(w, t->len_extra, lf_len_extra[t->len_code]); + if (r != LF_OK) { + return r; + } + r = lf_bwriter_put_code(w, dist_code[t->dist_code], + dist_len[t->dist_code]); + if (r != LF_OK) { + return r; + } + r = lf_bwriter_put(w, t->dist_extra, lf_dist_extra[t->dist_code]); + if (r != LF_OK) { + return r; + } + } + } + r = lf_bwriter_put_code(w, lit_code[LF_EOB], lit_len[LF_EOB]); + if (r != LF_OK) { + return r; + } + return lf_bwriter_align(w); +} + +lf_result lf_block_encode(lf_options *opt, lf_match *m, const uint8_t *src, + size_t base, size_t len, size_t total, int final, + lf_buf *out) +{ + lf_token *tokens; + uint32_t freq_lit[LF_NUM_LITLEN]; + uint32_t freq_dist[LF_NUM_DIST]; + uint8_t lit_len[LF_NUM_LITLEN]; + uint8_t dist_len[LF_NUM_DIST]; + uint32_t lit_code[LF_NUM_LITLEN]; + uint32_t dist_code[LF_NUM_DIST]; + uint32_t cl_code[LF_CL_SYMBOLS]; + uint8_t cl_len[LF_CL_SYMBOLS]; + size_t ntok; + uint16_t nlit; + uint16_t ndist; + lf_buf tmp; + lf_bwriter w; + lf_result r; + + if (opt == NULL || m == NULL || src == NULL || out == NULL) { + return LF_ERR_PARAM; + } + if (len == 0 || base + len > total) { + return LF_ERR_PARAM; + } + + memset(freq_lit, 0, sizeof(freq_lit)); + memset(freq_dist, 0, sizeof(freq_dist)); + + tokens = (lf_token *)malloc(len * sizeof(lf_token)); + if (tokens == NULL) { + return LF_ERR_ALLOC; + } + + ntok = 0; + { + size_t pos = 0; + + while (pos < len) { + unsigned ml = 0; + uint32_t md = 0; + + if (len - pos >= LF_MIN_MATCH) { + r = lf_match_search(m, src, base + pos, total, &ml, &md); + if (r != LF_OK) { + free(tokens); + return r; + } + } + if (ml >= LF_MIN_MATCH && md >= 1 && md <= opt->window_size) { + lf_token *t = &tokens[ntok++]; + unsigned le; + unsigned de; + + if (ml > len - pos) { + ml = (unsigned)(len - pos); + } + if (ml < LF_MIN_MATCH) { + t->kind = 0; + t->lit = src[base + pos]; + t->len_code = 0; + t->len_extra = 0; + t->dist_code = 0; + t->dist_extra = 0; + freq_lit[src[base + pos]]++; + lf_match_insert(m, src, base + pos, total); + pos++; + continue; + } + t->kind = 1; + t->lit = 0; + t->len_code = (uint16_t)lf_len_lookup(ml, &le); + t->len_extra = (uint16_t)le; + t->dist_code = (uint16_t)lf_dist_lookup(md, &de); + t->dist_extra = (uint16_t)de; + freq_lit[LF_FIRST_LEN + t->len_code]++; + freq_dist[t->dist_code]++; + { + unsigned k; + + for (k = 0; k < ml; k++) { + lf_match_insert(m, src, base + pos + k, total); + } + } + pos += ml; + } else { + lf_token *t = &tokens[ntok++]; + + t->kind = 0; + t->lit = src[base + pos]; + t->len_code = 0; + t->len_extra = 0; + t->dist_code = 0; + t->dist_extra = 0; + freq_lit[src[base + pos]]++; + lf_match_insert(m, src, base + pos, total); + pos++; + } + } + } + + freq_lit[LF_EOB]++; + + lf_huff_lengths(lit_len, freq_lit, LF_NUM_LITLEN); + lf_huff_lengths(dist_len, freq_dist, LF_NUM_DIST); + lf_huff_canonical(lit_code, lit_len, LF_NUM_LITLEN); + lf_huff_canonical(dist_code, dist_len, LF_NUM_DIST); + + nlit = LF_NUM_LITLEN; + while (nlit > LF_FIRST_LEN && lit_len[nlit - 1] == 0) { + nlit--; + } + ndist = LF_NUM_DIST; + while (ndist > 1 && dist_len[ndist - 1] == 0) { + ndist--; + } + + lf_buf_init(&tmp); + lf_bwriter_init(&w, &tmp); + lf_cl_build(cl_code, cl_len); + + r = lf_buf_append_u8(&tmp, + (uint8_t)(LF_BLOCK_TYPE_HUFFMAN | + (final ? LF_BLOCK_FLAG_FINAL : 0))); + if (r != LF_OK) { + goto done; + } + r = lf_buf_put_u16le(&tmp, nlit); + if (r != LF_OK) { + goto done; + } + r = lf_buf_put_u16le(&tmp, ndist); + if (r != LF_OK) { + goto done; + } + r = lf_cl_write(&w, lit_len, nlit, cl_code, cl_len); + if (r != LF_OK) { + goto done; + } + r = lf_cl_write(&w, dist_len, ndist, cl_code, cl_len); + if (r != LF_OK) { + goto done; + } + r = lf_write_tokens(&w, tokens, ntok, lit_code, lit_len, dist_code, dist_len); + if (r != LF_OK) { + goto done; + } + + if (tmp.len < 5 + len) { + r = lf_buf_append(out, tmp.data, tmp.len); + } else { + r = lf_buf_append_u8(out, + (uint8_t)(LF_BLOCK_TYPE_STORED | + (final ? LF_BLOCK_FLAG_FINAL : 0))); + if (r == LF_OK) { + r = lf_buf_put_u32le(out, (uint32_t)len); + } + if (r == LF_OK) { + r = lf_buf_append(out, src + base, len); + } + } + +done: + lf_buf_free(&tmp); + free(tokens); + return r; +} + +lf_result lf_block_decode(lf_breader *r, lf_buf *out, size_t max_out, + uint32_t window, int *final) +{ + uint8_t hdr; + unsigned type; + + if (r == NULL || out == NULL || final == NULL) { + return LF_ERR_PARAM; + } + *final = 0; + if (!lf_breader_read_bytes(r, &hdr, 1)) { + return LF_ERR_TRUNCATED; + } + type = (unsigned)(hdr & 0x03u); + if ((hdr & 0xF8u) != 0) { + return LF_ERR_INVALID_DATA; + } + *final = (hdr & LF_BLOCK_FLAG_FINAL) != 0; + + if (type == LF_BLOCK_TYPE_STORED) { + uint8_t lb[4]; + uint32_t slen; + lf_result res; + + if (!lf_breader_read_bytes(r, lb, 4)) { + return LF_ERR_TRUNCATED; + } + slen = (uint32_t)lb[0] | ((uint32_t)lb[1] << 8) | + ((uint32_t)lb[2] << 16) | ((uint32_t)lb[3] << 24); + if (slen == 0) { + return LF_ERR_INVALID_DATA; + } + if (out->len > max_out || (size_t)slen > max_out - out->len) { + return LF_ERR_BAD_LENGTH; + } + if (r->pos > r->len || (size_t)slen > r->len - r->pos) { + return LF_ERR_TRUNCATED; + } + res = lf_buf_reserve(out, slen); + if (res != LF_OK) { + return res; + } + memcpy(out->data + out->len, r->data + r->pos, slen); + out->len += slen; + r->pos += slen; + return LF_OK; + } + + if (type == LF_BLOCK_TYPE_HUFFMAN) { + uint8_t nb[2]; + uint16_t nlit; + uint16_t ndist; + uint8_t lit_len[LF_NUM_LITLEN]; + uint8_t dist_len[LF_NUM_DIST]; + lf_huff_dec lit_dec; + lf_huff_dec dist_dec; + lf_result res; + + if (!lf_breader_read_bytes(r, nb, 2)) { + return LF_ERR_TRUNCATED; + } + nlit = (uint16_t)(nb[0] | (uint16_t)(nb[1] << 8)); + if (!lf_breader_read_bytes(r, nb, 2)) { + return LF_ERR_TRUNCATED; + } + ndist = (uint16_t)(nb[0] | (uint16_t)(nb[1] << 8)); + if (nlit < LF_FIRST_LEN || nlit > LF_NUM_LITLEN) { + return LF_ERR_INVALID_DATA; + } + if (ndist < 1 || ndist > LF_NUM_DIST) { + return LF_ERR_INVALID_DATA; + } + + memset(lit_len, 0, sizeof(lit_len)); + memset(dist_len, 0, sizeof(dist_len)); + res = lf_cl_read(r, lit_len, nlit); + if (res != LF_OK) { + return res; + } + res = lf_cl_read(r, dist_len, ndist); + if (res != LF_OK) { + return res; + } + + lf_huff_dec_init(&lit_dec, lit_len, LF_NUM_LITLEN); + if (!lit_dec.valid) { + return LF_ERR_INVALID_DATA; + } + lf_huff_dec_init(&dist_dec, dist_len, LF_NUM_DIST); + + for (;;) { + int sym = lf_huff_dec_symbol(&lit_dec, r); + + if (sym < 0) { + return LF_ERR_TRUNCATED; + } + if (sym < 256) { + if (out->len >= max_out) { + return LF_ERR_BAD_LENGTH; + } + res = lf_buf_append_u8(out, (uint8_t)sym); + if (res != LF_OK) { + return res; + } + } else if (sym == LF_EOB) { + lf_breader_align(r); + return LF_OK; + } else if (sym <= (int)(LF_FIRST_LEN + LF_LEN_CODES - 1u)) { + unsigned code = (unsigned)(sym - LF_FIRST_LEN); + uint32_t extra = lf_breader_get(r, lf_len_extra[code]); + uint32_t length; + uint32_t dist; + uint32_t dextra; + int dcode; + size_t start; + size_t i; + + if (!lf_breader_ok(r)) { + return LF_ERR_TRUNCATED; + } + length = (uint32_t)lf_len_base[code] + extra; + dcode = lf_huff_dec_symbol(&dist_dec, r); + if (dcode < 0 || (unsigned)dcode >= LF_DIST_CODES) { + return LF_ERR_INVALID_DATA; + } + if (!dist_dec.valid) { + return LF_ERR_INVALID_DATA; + } + dextra = lf_breader_get(r, lf_dist_extra[dcode]); + if (!lf_breader_ok(r)) { + return LF_ERR_TRUNCATED; + } + dist = (uint32_t)lf_dist_base[dcode] + dextra; + if (dist == 0 || dist > out->len || dist > window) { + return LF_ERR_INVALID_DATA; + } + if (out->len > max_out || (size_t)length > max_out - out->len) { + return LF_ERR_BAD_LENGTH; + } + res = lf_buf_reserve(out, length); + if (res != LF_OK) { + return res; + } + start = out->len; + for (i = 0; i < length; i++) { + out->data[start + i] = out->data[start + i - dist]; + } + out->len += length; + } else { + return LF_ERR_INVALID_DATA; + } + } + } + + return LF_ERR_INVALID_DATA; +} + diff --git a/src/lf_block.h b/src/lf_block.h new file mode 100644 index 0000000..6ed1ab3 --- /dev/null +++ b/src/lf_block.h @@ -0,0 +1,32 @@ +#ifndef LF_BLOCK_H +#define LF_BLOCK_H + +#include "lf_bitio.h" +#include "lf_common.h" +#include "lf_match.h" + +/* + * Block-level compression. + * A block holds one unit of Huffman-compressed data. Blocks are independent: + * each carries its own code-length description and token stream. A final flag + * marks the last block of a stream. + * + * The encoder tokenizes `src[base .. base+len)` with the caller-owned match + * finder, so matches may reference earlier blocks. It then emits either a + * stored block or a Huffman block, whichever is smaller. + */ + +/* Encodes one block. `src` is the full input buffer, `base` is the block + start, `len` is the block length, and `total` is the full input length. + `final` marks the last block of the stream. */ +LF_EXPORT lf_result lf_block_encode(lf_options *opt, lf_match *m, + const uint8_t *src, size_t base, size_t len, + size_t total, int final, lf_buf *out); + +/* Decodes one block from `r` into `out`. + `max_out` bounds the output length and `window` bounds the match distance. + Sets `*final` from the block header. */ +LF_EXPORT lf_result lf_block_decode(lf_breader *r, lf_buf *out, size_t max_out, + uint32_t window, int *final); + +#endif /* LF_BLOCK_H */ diff --git a/src/lf_buf.c b/src/lf_buf.c index dd147e8..090b7ed 100644 --- a/src/lf_buf.c +++ b/src/lf_buf.c @@ -76,6 +76,15 @@ void lf_buf_clear(lf_buf *b) b->len = 0; } +lf_result lf_buf_put_u16le(lf_buf *b, uint16_t v) +{ + uint8_t t[2]; + + t[0] = (uint8_t)(v & 0xFFu); + t[1] = (uint8_t)((v >> 8u) & 0xFFu); + return lf_buf_append(b, t, sizeof(t)); +} + lf_result lf_buf_put_u32le(lf_buf *b, uint32_t v) { uint8_t t[4]; diff --git a/src/lf_buf.h b/src/lf_buf.h index 7fa7a01..1cd2be3 100644 --- a/src/lf_buf.h +++ b/src/lf_buf.h @@ -21,6 +21,8 @@ LF_EXPORT lf_result lf_buf_append_u8(lf_buf *b, uint8_t v); LF_EXPORT void lf_buf_clear(lf_buf *b); +LF_EXPORT lf_result lf_buf_put_u16le(lf_buf *b, uint16_t v); + LF_EXPORT lf_result lf_buf_put_u32le(lf_buf *b, uint32_t v); LF_EXPORT lf_result lf_buf_put_u64le(lf_buf *b, uint64_t v); diff --git a/src/lf_common.c b/src/lf_common.c new file mode 100644 index 0000000..4bbb7d4 --- /dev/null +++ b/src/lf_common.c @@ -0,0 +1,43 @@ +#include "lf_common.h" + +const char *lf_version_string(void) +{ + return "1.1.0"; +} + +void lf_options_default(lf_options *opt) +{ + if (opt == NULL) { + return; + } + opt->block_size = LF_DEFAULT_BLOCK; + opt->window_size = LF_DEFAULT_WINDOW; + opt->max_chain = LF_DEFAULT_MAX_CHAIN; +} + +const char *lf_result_name(lf_result r) +{ + switch (r) { + case LF_OK: + return "LF_OK"; + case LF_ERR_ALLOC: + return "LF_ERR_ALLOC"; + case LF_ERR_PARAM: + return "LF_ERR_PARAM"; + case LF_ERR_IO: + return "LF_ERR_IO"; + case LF_ERR_BAD_MAGIC: + return "LF_ERR_BAD_MAGIC"; + case LF_ERR_UNSUPPORTED_VERSION: + return "LF_ERR_UNSUPPORTED_VERSION"; + case LF_ERR_TRUNCATED: + return "LF_ERR_TRUNCATED"; + case LF_ERR_INVALID_DATA: + return "LF_ERR_INVALID_DATA"; + case LF_ERR_BAD_LENGTH: + return "LF_ERR_BAD_LENGTH"; + case LF_ERR_BUFFER_FULL: + return "LF_ERR_BUFFER_FULL"; + } + return "LF_ERR_UNKNOWN"; +} diff --git a/src/lf_common.h b/src/lf_common.h index 90bb643..de6e339 100644 --- a/src/lf_common.h +++ b/src/lf_common.h @@ -13,7 +13,7 @@ #endif #define LF_VERSION_MAJOR 1 -#define LF_VERSION_MINOR 0 +#define LF_VERSION_MINOR 1 #define LF_VERSION_PATCH 0 #define LF_MAGIC_0 0x4Cu /* 'L' */ @@ -37,7 +37,7 @@ #define LF_MAX_CODE_LEN 15u -/* Position 0 means "no previous position with this hash". */ +/* Chain positions are stored one-based. 0 means "no previous position". */ #define LF_LZ_NONE 0u typedef enum lf_result { diff --git a/src/lf_match.c b/src/lf_match.c new file mode 100644 index 0000000..5db9518 --- /dev/null +++ b/src/lf_match.c @@ -0,0 +1,144 @@ +#include "lf_match.h" + +#include +#include + +static uint32_t lf_match_hash(const uint8_t *p) +{ + uint32_t h = ((uint32_t)p[0] << 16) | ((uint32_t)p[1] << 8) | (uint32_t)p[2]; + + h ^= h >> 15; + h *= 0x1E35A7BDu; + h ^= h >> 13; + return h; +} + +lf_result lf_match_init(lf_match *m, uint32_t window, uint32_t max_chain) +{ + uint32_t table_size; + + if (m == NULL) { + return LF_ERR_PARAM; + } + m->head = NULL; + m->prev = NULL; + m->window = 0; + m->max_chain = 0; + m->hash_mask = 0; + + if (window < LF_MIN_MATCH || window > LF_MAX_WINDOW) { + return LF_ERR_PARAM; + } + if (max_chain == 0) { + return LF_ERR_PARAM; + } + + table_size = (uint32_t)1 << LF_MATCH_HASH_BITS; + m->head = (uint32_t *)calloc(table_size, sizeof(uint32_t)); + m->prev = (uint32_t *)calloc(window, sizeof(uint32_t)); + if (m->head == NULL || m->prev == NULL) { + free(m->head); + free(m->prev); + m->head = NULL; + m->prev = NULL; + return LF_ERR_ALLOC; + } + + m->window = window; + m->max_chain = max_chain; + m->hash_mask = table_size - 1u; + return LF_OK; +} + +void lf_match_free(lf_match *m) +{ + if (m == NULL) { + return; + } + free(m->head); + free(m->prev); + m->head = NULL; + m->prev = NULL; + m->window = 0; + m->max_chain = 0; + m->hash_mask = 0; +} + +void lf_match_reset(lf_match *m) +{ + uint32_t table_size; + + if (m == NULL) { + return; + } + table_size = m->hash_mask + 1u; + memset(m->head, 0, (size_t)table_size * sizeof(uint32_t)); + memset(m->prev, 0, (size_t)m->window * sizeof(uint32_t)); +} + +lf_result lf_match_search(lf_match *m, const uint8_t *src, size_t pos, + size_t src_len, unsigned *match_len, uint32_t *match_dist) +{ + uint32_t hash; + uint32_t cand; + unsigned chain; + unsigned best = 0; + uint32_t best_dist = 0; + + if (m == NULL || src == NULL || match_len == NULL || match_dist == NULL) { + return LF_ERR_PARAM; + } + *match_len = 0; + *match_dist = 0; + if (pos + LF_MIN_MATCH > src_len) { + return LF_OK; + } + + hash = lf_match_hash(src + pos); + cand = m->head[hash & m->hash_mask]; + for (chain = 0; cand != LF_LZ_NONE && chain < m->max_chain; chain++) { + uint32_t c = cand - 1u; + size_t k; + + if (c >= pos) { + break; + } + if (pos - c > m->window) { + break; + } + k = 0; + while (k < LF_MAX_MATCH && pos + k < src_len && + src[c + k] == src[pos + k]) { + k++; + } + if (k > best) { + best = (unsigned)k; + best_dist = (uint32_t)(pos - c); + if (best == LF_MAX_MATCH) { + break; + } + } + cand = m->prev[c % m->window]; + } + + *match_len = best; + *match_dist = best_dist; + return LF_OK; +} + +void lf_match_insert(lf_match *m, const uint8_t *src, size_t pos, size_t src_len) +{ + uint32_t hash; + uint32_t slot; + + if (m == NULL || src == NULL) { + return; + } + if (pos + LF_MIN_MATCH > src_len) { + return; + } + hash = lf_match_hash(src + pos); + slot = (uint32_t)(pos % m->window); + m->prev[slot] = m->head[hash & m->hash_mask]; + m->head[hash & m->hash_mask] = (uint32_t)(pos + 1u); +} diff --git a/src/lf_match.h b/src/lf_match.h new file mode 100644 index 0000000..60a2ecb --- /dev/null +++ b/src/lf_match.h @@ -0,0 +1,41 @@ +#ifndef LF_MATCH_H +#define LF_MATCH_H + +#include "lf_common.h" + +/* + * An LZ77 hash-chain match finder. + * It indexes 3-byte sequences in a sliding window and finds the longest + * previous occurrence of the data at a given position. + */ + +#define LF_MATCH_HASH_BITS 15u + +typedef struct lf_match { + uint32_t *head; + uint32_t *prev; + uint32_t window; + uint32_t max_chain; + uint32_t hash_mask; +} lf_match; + +/* Allocates a finder with the given window size and chain limit. */ +LF_EXPORT lf_result lf_match_init(lf_match *m, uint32_t window, uint32_t max_chain); + +LF_EXPORT void lf_match_free(lf_match *m); + +/* Clears all hash and history entries. */ +LF_EXPORT void lf_match_reset(lf_match *m); + +/* Searches for the longest match at `src + pos`. + Returns LF_OK. When no match exists, *match_len is 0 and *match_dist is 0. + Matches never exceed LF_MAX_MATCH bytes. */ +LF_EXPORT lf_result lf_match_search(lf_match *m, const uint8_t *src, size_t pos, + size_t src_len, unsigned *match_len, + uint32_t *match_dist); + +/* Records `src + pos` so later searches can find it. */ +LF_EXPORT void lf_match_insert(lf_match *m, const uint8_t *src, size_t pos, + size_t src_len); + +#endif /* LF_MATCH_H */ diff --git a/src/lf_stream.c b/src/lf_stream.c new file mode 100644 index 0000000..77f7712 --- /dev/null +++ b/src/lf_stream.c @@ -0,0 +1,251 @@ +#include "lf_stream.h" + +#include "lf_adler32.h" +#include "lf_block.h" +#include "lf_match.h" + +#include +#include + +#define LF_HEADER_SIZE 24u +#define LF_MIN_BLOCK 256u +#define LF_PREALLOC_LIMIT ((size_t)1 << 20) + +static lf_result lf_options_check(const lf_options *in, lf_options *out) +{ + if (in == NULL) { + lf_options_default(out); + return LF_OK; + } + *out = *in; + if (out->window_size == 0) { + out->window_size = LF_DEFAULT_WINDOW; + } + if (out->window_size < LF_MIN_MATCH || out->window_size > LF_MAX_WINDOW) { + return LF_ERR_PARAM; + } + if (out->block_size == 0) { + out->block_size = LF_DEFAULT_BLOCK; + } + if (out->block_size < LF_MIN_BLOCK || out->block_size > LF_MAX_BLOCK) { + return LF_ERR_PARAM; + } + if (out->max_chain == 0) { + out->max_chain = LF_DEFAULT_MAX_CHAIN; + } + return LF_OK; +} + +static void lf_put_u64le(uint8_t *p, uint64_t v) +{ + unsigned i; + + for (i = 0; i < 8; i++) { + p[i] = (uint8_t)(v & 0xFFu); + v >>= 8; + } +} + +static void lf_put_u32le(uint8_t *p, uint32_t v) +{ + unsigned i; + + for (i = 0; i < 4; i++) { + p[i] = (uint8_t)(v & 0xFFu); + v >>= 8; + } +} + +static uint64_t lf_get_u64le(const uint8_t *p) +{ + uint64_t v = 0; + unsigned i; + + for (i = 0; i < 8; i++) { + v |= (uint64_t)p[i] << (8 * i); + } + return v; +} + +static uint32_t lf_get_u32le(const uint8_t *p) +{ + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | + ((uint32_t)p[3] << 24); +} + +static lf_result lf_write_header(lf_buf *out, uint64_t size, uint32_t adler, + uint32_t window) +{ + uint8_t h[LF_HEADER_SIZE]; + + memset(h, 0, sizeof(h)); + h[0] = LF_MAGIC_0; + h[1] = LF_MAGIC_1; + h[2] = LF_MAGIC_2; + h[3] = (uint8_t)LF_FORMAT_VERSION; + lf_put_u64le(h + 4, size); + lf_put_u32le(h + 12, adler); + lf_put_u32le(h + 16, window); + return lf_buf_append(out, h, sizeof(h)); +} + +static lf_result lf_read_header(lf_breader *r, uint64_t *size, uint32_t *adler, + uint32_t *window) +{ + uint8_t h[LF_HEADER_SIZE]; + + if (!lf_breader_read_bytes(r, h, sizeof(h))) { + return LF_ERR_TRUNCATED; + } + if (h[0] != LF_MAGIC_0 || h[1] != LF_MAGIC_1 || h[2] != LF_MAGIC_2) { + return LF_ERR_BAD_MAGIC; + } + if (h[3] != LF_FORMAT_VERSION) { + return LF_ERR_UNSUPPORTED_VERSION; + } + if (lf_get_u32le(h + 20) != 0) { + return LF_ERR_INVALID_DATA; + } + *size = lf_get_u64le(h + 4); + *adler = lf_get_u32le(h + 12); + *window = lf_get_u32le(h + 16); + if (*window < LF_MIN_MATCH || *window > LF_MAX_WINDOW) { + return LF_ERR_INVALID_DATA; + } + return LF_OK; +} + +size_t lf_compress_bound(size_t src_len) +{ + size_t bits; + size_t blocks; + + if (src_len > (size_t)((uint64_t)-1) / 15) { + return (size_t)((uint64_t)-1); + } + bits = src_len * 15 + 8; + blocks = src_len / LF_MIN_BLOCK + 1; + return LF_HEADER_SIZE + bits / 8 + blocks * 250 + 8; +} + +lf_result lf_compress(const lf_options *opt, const void *src, size_t src_len, + lf_buf *out) +{ + lf_options o; + lf_match m; + uint32_t adler; + lf_result res; + size_t pos = 0; + int have_match = 0; + + if (out == NULL) { + return LF_ERR_PARAM; + } + if (src_len > 0 && src == NULL) { + return LF_ERR_PARAM; + } + if (src_len > (size_t)0xFFFFFFFFu) { + return LF_ERR_PARAM; + } + res = lf_options_check(opt, &o); + if (res != LF_OK) { + return res; + } + + adler = lf_adler32(src, src_len); + res = lf_write_header(out, (uint64_t)src_len, adler, o.window_size); + if (res != LF_OK) { + return res; + } + if (src_len == 0) { + return LF_OK; + } + + res = lf_match_init(&m, o.window_size, o.max_chain); + if (res != LF_OK) { + return res; + } + have_match = 1; + + while (pos < src_len) { + size_t blen = src_len - pos; + int final; + + if (blen > o.block_size) { + blen = o.block_size; + } + final = (pos + blen == src_len); + res = lf_block_encode(&o, &m, (const uint8_t *)src, pos, blen, src_len, + final, out); + if (res != LF_OK) { + break; + } + pos += blen; + } + + if (have_match) { + lf_match_free(&m); + } + return res; +} + +lf_result lf_decompress(const void *src, size_t src_len, lf_buf *out) +{ + lf_breader r; + uint64_t decl_size; + uint32_t adler; + uint32_t window; + lf_result res; + int final; + int saw_block = 0; + + if (src == NULL || out == NULL) { + return LF_ERR_PARAM; + } + if (src_len < LF_HEADER_SIZE) { + return LF_ERR_TRUNCATED; + } + + lf_breader_init(&r, src, src_len); + res = lf_read_header(&r, &decl_size, &adler, &window); + if (res != LF_OK) { + return res; + } + if (decl_size > (uint64_t)SIZE_MAX) { + return LF_ERR_BAD_LENGTH; + } + + if (decl_size > 0) { + size_t prealloc = (size_t)decl_size; + + if (prealloc > LF_PREALLOC_LIMIT) { + prealloc = LF_PREALLOC_LIMIT; + } + (void)lf_buf_reserve(out, prealloc); + } + + while (r.pos < r.len) { + res = lf_block_decode(&r, out, (size_t)decl_size, window, &final); + if (res != LF_OK) { + return res; + } + saw_block = 1; + if (final) { + break; + } + } + + if (decl_size > 0 && !saw_block) { + return LF_ERR_TRUNCATED; + } + if (out->len != (size_t)decl_size) { + return LF_ERR_BAD_LENGTH; + } + if (lf_adler32(out->data, out->len) != adler) { + return LF_ERR_INVALID_DATA; + } + if (lf_breader_bytes_read(&r) != src_len) { + return LF_ERR_INVALID_DATA; + } + return LF_OK; +} diff --git a/src/lf_stream.h b/src/lf_stream.h new file mode 100644 index 0000000..1e5918f --- /dev/null +++ b/src/lf_stream.h @@ -0,0 +1,25 @@ +#ifndef LF_STREAM_H +#define LF_STREAM_H + +#include "lf_buf.h" +#include "lf_common.h" + +/* + * Whole-stream compression. + * A compressed stream starts with a fixed header and continues with one or + * more blocks. The header records the original size, an Adler-32 checksum, + * and the window size used during compression. + */ + +/* Compresses `src_len` bytes from `src` into `out`. + `opt` may be NULL to use the defaults. The stream is fully self-contained. */ +LF_EXPORT lf_result lf_compress(const lf_options *opt, const void *src, + size_t src_len, lf_buf *out); + +/* Decompresses a stream into `out`. Rejects corrupt or truncated streams. */ +LF_EXPORT lf_result lf_decompress(const void *src, size_t src_len, lf_buf *out); + +/* Returns an upper bound on the compressed size of `src_len` bytes. */ +LF_EXPORT size_t lf_compress_bound(size_t src_len); + +#endif /* LF_STREAM_H */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..41e2d9e --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,14 @@ +set(UNITY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/unity) + +file(GLOB TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/test_*.c) + +foreach(test_src ${TEST_SOURCES}) + get_filename_component(test_name ${test_src} NAME_WE) + add_executable(${test_name} ${test_src} ${UNITY_DIR}/unity.c) + target_include_directories(${test_name} PRIVATE ${UNITY_DIR}) + target_link_libraries(${test_name} PRIVATE leaflitter::leaflitter) + if(NOT MSVC) + target_compile_options(${test_name} PRIVATE -Wall -Wextra) + endif() + add_test(NAME ${test_name} COMMAND ${test_name}) +endforeach() diff --git a/tests/test_adler32.c b/tests/test_adler32.c new file mode 100644 index 0000000..c54319f --- /dev/null +++ b/tests/test_adler32.c @@ -0,0 +1,92 @@ +#include "unity.h" + +#include "lf_adler32.h" + +#include + +void setUp(void) +{ +} + +void tearDown(void) +{ +} + +static uint32_t rng_state = 7u; + +static uint32_t next_rng(void) +{ + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; + return rng_state; +} + +static uint32_t reference_adler32(const uint8_t *data, size_t len) +{ + uint32_t a = 1; + uint32_t b = 0; + size_t i; + + for (i = 0; i < len; i++) { + a = (a + data[i]) % 65521u; + b = (b + a) % 65521u; + } + return (b << 16) | a; +} + +static void test_empty_is_one(void) +{ + TEST_ASSERT_EQUAL_UINT(1, lf_adler32(NULL, 0)); +} + +static void test_known_vector(void) +{ + TEST_ASSERT_EQUAL_UINT(0x11E60398u, lf_adler32("Wikipedia", 9)); +} + +static void test_single_bytes(void) +{ + uint8_t one[1] = {0x01}; + + TEST_ASSERT_EQUAL_UINT(reference_adler32(one, 1), lf_adler32(one, 1)); + TEST_ASSERT_EQUAL_UINT(0x00020002u, lf_adler32(one, 1)); +} + +static void test_matches_reference(void) +{ + uint8_t data[4096]; + size_t i; + unsigned iter; + + for (iter = 0; iter < 8; iter++) { + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)next_rng(); + } + TEST_ASSERT_EQUAL_UINT(reference_adler32(data, sizeof(data)), + lf_adler32(data, sizeof(data))); + } +} + +static void test_lengths_agree(void) +{ + uint8_t data[100]; + size_t i; + + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)(i * 7u); + TEST_ASSERT_EQUAL_UINT(reference_adler32(data, i + 1), + lf_adler32(data, i + 1)); + } +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_empty_is_one); + RUN_TEST(test_known_vector); + RUN_TEST(test_single_bytes); + RUN_TEST(test_matches_reference); + RUN_TEST(test_lengths_agree); + return UNITY_END(); +} diff --git a/tests/test_bitio.c b/tests/test_bitio.c new file mode 100644 index 0000000..170a637 --- /dev/null +++ b/tests/test_bitio.c @@ -0,0 +1,178 @@ +#include "unity.h" + +#include "lf_bitio.h" +#include "lf_buf.h" + +#include + +void setUp(void) +{ +} + +void tearDown(void) +{ +} + +static void test_put_get_roundtrip(void) +{ + lf_buf b; + lf_bwriter w; + lf_breader r; + unsigned i; + + lf_buf_init(&b); + lf_bwriter_init(&w, &b); + + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put(&w, 0b101u, 3)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put(&w, 0b01u, 2)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put(&w, 0x1F3u, 9)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_align(&w)); + + lf_breader_init(&r, b.data, b.len); + TEST_ASSERT_EQUAL_UINT(0b101u, lf_breader_get(&r, 3)); + TEST_ASSERT_EQUAL_UINT(0b01u, lf_breader_get(&r, 2)); + TEST_ASSERT_EQUAL_UINT(0x1F3u, lf_breader_get(&r, 9)); + TEST_ASSERT_TRUE(lf_breader_ok(&r)); + lf_breader_align(&r); + TEST_ASSERT_TRUE(lf_breader_finished(&r)); + + for (i = 0; i < b.len; i++) { + b.data[i] = 0; + } + lf_buf_free(&b); +} + +static void test_msb_first_order(void) +{ + lf_buf b; + lf_bwriter w; + lf_breader r; + + lf_buf_init(&b); + lf_bwriter_init(&w, &b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put(&w, 0xAAu, 8)); + lf_bwriter_align(&w); + + TEST_ASSERT_EQUAL_UINT(1, b.len); + TEST_ASSERT_EQUAL_UINT(0xAAu, b.data[0]); + + lf_breader_init(&r, b.data, b.len); + TEST_ASSERT_EQUAL_UINT(0xAAu, lf_breader_get(&r, 8)); + lf_buf_free(&b); +} + +static void test_bit_boundary_across_bytes(void) +{ + lf_buf b; + lf_bwriter w; + lf_breader r; + + lf_buf_init(&b); + lf_bwriter_init(&w, &b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put(&w, 0xFFu, 4)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put(&w, 0xFFu, 4)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put(&w, 0x01u, 1)); + lf_bwriter_align(&w); + + TEST_ASSERT_EQUAL_UINT(2, b.len); + TEST_ASSERT_EQUAL_UINT(0xFFu, b.data[0]); + TEST_ASSERT_EQUAL_UINT(0x80u, b.data[1]); + + lf_breader_init(&r, b.data, b.len); + TEST_ASSERT_EQUAL_UINT(0x0Fu, lf_breader_get(&r, 4)); + TEST_ASSERT_EQUAL_UINT(0x0Fu, lf_breader_get(&r, 4)); + TEST_ASSERT_EQUAL_UINT(0x01u, lf_breader_get(&r, 1)); + TEST_ASSERT_TRUE(lf_breader_ok(&r)); + lf_buf_free(&b); +} + +static void test_truncated_read_fails(void) +{ + uint8_t data[1] = {0x00}; + lf_breader r; + + lf_breader_init(&r, data, sizeof(data)); + TEST_ASSERT_EQUAL_UINT(0, lf_breader_get(&r, 8)); + TEST_ASSERT_TRUE(lf_breader_ok(&r)); + TEST_ASSERT_EQUAL_UINT(0, lf_breader_get(&r, 1)); + TEST_ASSERT_FALSE(lf_breader_ok(&r)); +} + +static void test_align_drops_partial_byte(void) +{ + lf_buf b; + lf_bwriter w; + lf_breader r; + + lf_buf_init(&b); + lf_bwriter_init(&w, &b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put(&w, 0b101u, 3)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_align(&w)); + TEST_ASSERT_EQUAL_UINT(1, b.len); + + lf_breader_init(&r, b.data, b.len); + TEST_ASSERT_EQUAL_UINT(0b101u, lf_breader_get(&r, 3)); + lf_breader_align(&r); + TEST_ASSERT_EQUAL_UINT(1, lf_breader_bytes_read(&r)); + lf_buf_free(&b); +} + +static void test_read_bytes_whole(void) +{ + uint8_t data[6] = {1, 2, 3, 4, 5, 6}; + uint8_t got[4]; + lf_breader r; + + lf_breader_init(&r, data, sizeof(data)); + TEST_ASSERT_TRUE(lf_breader_read_bytes(&r, got, 4)); + TEST_ASSERT_EQUAL_UINT(4, lf_breader_bytes_read(&r)); + TEST_ASSERT_EQUAL_UINT(0, memcmp(got, data, 4)); + TEST_ASSERT_FALSE(lf_breader_read_bytes(&r, got, 4)); +} + +static void test_long_codes_roundtrip(void) +{ + lf_buf b; + lf_bwriter w; + lf_breader r; + unsigned n = 700; + uint8_t vals[700]; + uint8_t nbits[700]; + unsigned i; + unsigned s = 1u; + + for (i = 0; i < n; i++) { + s = s * 1103515245u + 12345u; + nbits[i] = (uint8_t)(1u + (s >> 16) % 16u); + vals[i] = (uint8_t)(s & ((1u << (nbits[i] < 8 ? nbits[i] : 8)) - 1u)); + } + + lf_buf_init(&b); + lf_bwriter_init(&w, &b); + for (i = 0; i < n; i++) { + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put(&w, vals[i], nbits[i])); + } + lf_bwriter_align(&w); + + lf_breader_init(&r, b.data, b.len); + for (i = 0; i < n; i++) { + TEST_ASSERT_EQUAL_UINT(vals[i], lf_breader_get(&r, nbits[i])); + } + TEST_ASSERT_TRUE(lf_breader_ok(&r)); + lf_breader_align(&r); + TEST_ASSERT_TRUE(lf_breader_finished(&r)); + lf_buf_free(&b); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_put_get_roundtrip); + RUN_TEST(test_msb_first_order); + RUN_TEST(test_bit_boundary_across_bytes); + RUN_TEST(test_truncated_read_fails); + RUN_TEST(test_align_drops_partial_byte); + RUN_TEST(test_read_bytes_whole); + RUN_TEST(test_long_codes_roundtrip); + return UNITY_END(); +} diff --git a/tests/test_block.c b/tests/test_block.c new file mode 100644 index 0000000..a206f5e --- /dev/null +++ b/tests/test_block.c @@ -0,0 +1,180 @@ +#include "unity.h" + +#include "lf_bitio.h" +#include "lf_block.h" +#include "lf_buf.h" +#include "lf_common.h" +#include "lf_match.h" + +#include + +void setUp(void) +{ +} + +void tearDown(void) +{ +} + +static uint32_t rng_state = 42u; + +static uint32_t next_rng(void) +{ + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; + return rng_state; +} + +static void block_roundtrip(const uint8_t *data, size_t len, uint32_t window) +{ + lf_match m; + lf_options opt; + lf_buf packed; + lf_buf out; + lf_breader r; + int final = 0; + lf_result res; + + lf_options_default(&opt); + opt.window_size = window; + TEST_ASSERT_EQUAL_INT(LF_OK, lf_match_init(&m, opt.window_size, opt.max_chain)); + lf_buf_init(&packed); + lf_buf_init(&out); + + res = lf_block_encode(&opt, &m, data, 0, len, len, 1, &packed); + TEST_ASSERT_EQUAL_INT(LF_OK, res); + lf_match_free(&m); + TEST_ASSERT_TRUE(packed.len > 0); + + lf_breader_init(&r, packed.data, packed.len); + res = lf_block_decode(&r, &out, len, window, &final); + TEST_ASSERT_EQUAL_INT(LF_OK, res); + TEST_ASSERT_TRUE(final); + TEST_ASSERT_EQUAL_UINT(len, out.len); + TEST_ASSERT_EQUAL_INT(0, memcmp(out.data, data, len)); + TEST_ASSERT_TRUE(lf_breader_finished(&r)); + + lf_buf_free(&packed); + lf_buf_free(&out); +} + +static void test_single_literal(void) +{ + uint8_t data[1] = {0x2A}; + + block_roundtrip(data, 1, 256); +} + +static void test_short_repeat(void) +{ + const char *data = "abcabcabcabcabc"; + + block_roundtrip((const uint8_t *)data, strlen(data), 512); +} + +static void test_repeated_bytes(void) +{ + uint8_t data[1000]; + size_t i; + + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)(i % 7u); + } + block_roundtrip(data, sizeof(data), 4096); +} + +static void test_random_data_is_stored(void) +{ + uint8_t data[512]; + lf_match m; + lf_options opt; + lf_buf packed; + lf_buf out; + lf_breader r; + int final = 0; + size_t i; + + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)next_rng(); + } + + lf_options_default(&opt); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_match_init(&m, opt.window_size, opt.max_chain)); + lf_buf_init(&packed); + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_OK, + lf_block_encode(&opt, &m, data, 0, sizeof(data), + sizeof(data), 1, &packed)); + lf_match_free(&m); + + TEST_ASSERT_EQUAL_UINT(0, packed.data[0] & 0x03u); + + lf_breader_init(&r, packed.data, packed.len); + TEST_ASSERT_EQUAL_INT(LF_OK, + lf_block_decode(&r, &out, sizeof(data), opt.window_size, + &final)); + TEST_ASSERT_TRUE(final); + TEST_ASSERT_EQUAL_UINT(sizeof(data), out.len); + TEST_ASSERT_EQUAL_INT(0, memcmp(out.data, data, sizeof(data))); + + lf_buf_free(&packed); + lf_buf_free(&out); +} + +static void test_long_window_match(void) +{ + uint8_t data[4096]; + size_t i; + + for (i = 0; i < 2048; i++) { + data[i] = (uint8_t)(i % 251u); + } + memcpy(data + 2048, data, 2048); + block_roundtrip(data, sizeof(data), 4096); +} + +static void test_truncated_block_rejected(void) +{ + uint8_t data[64]; + lf_match m; + lf_options opt; + lf_buf packed; + lf_buf out; + lf_breader r; + int final = 0; + size_t i; + + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)(i * 3u); + } + + lf_options_default(&opt); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_match_init(&m, opt.window_size, opt.max_chain)); + lf_buf_init(&packed); + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_OK, + lf_block_encode(&opt, &m, data, 0, sizeof(data), + sizeof(data), 1, &packed)); + lf_match_free(&m); + TEST_ASSERT_TRUE(packed.len > 4); + + lf_breader_init(&r, packed.data, packed.len - 1); + TEST_ASSERT_NOT_EQUAL_INT(LF_OK, + lf_block_decode(&r, &out, sizeof(data), + opt.window_size, &final)); + lf_buf_free(&packed); + lf_buf_free(&out); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_single_literal); + RUN_TEST(test_short_repeat); + RUN_TEST(test_repeated_bytes); + RUN_TEST(test_random_data_is_stored); + RUN_TEST(test_long_window_match); + RUN_TEST(test_truncated_block_rejected); + return UNITY_END(); +} diff --git a/tests/test_buf.c b/tests/test_buf.c new file mode 100644 index 0000000..b134cd4 --- /dev/null +++ b/tests/test_buf.c @@ -0,0 +1,135 @@ +#include "unity.h" + +#include "lf_buf.h" + +#include + +void setUp(void) +{ +} + +void tearDown(void) +{ +} + +static void test_init_is_empty(void) +{ + lf_buf b; + + lf_buf_init(&b); + TEST_ASSERT_NULL(b.data); + TEST_ASSERT_EQUAL_UINT(0, b.len); + TEST_ASSERT_EQUAL_UINT(0, b.cap); + lf_buf_free(&b); +} + +static void test_append_grows(void) +{ + lf_buf b; + uint8_t data[1000]; + size_t i; + + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)(i & 0xFFu); + } + + lf_buf_init(&b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_buf_append(&b, data, sizeof(data))); + TEST_ASSERT_EQUAL_UINT(sizeof(data), b.len); + TEST_ASSERT_EQUAL_UINT(0, memcmp(b.data, data, sizeof(data))); + TEST_ASSERT_TRUE(b.cap >= b.len); + lf_buf_free(&b); +} + +static void test_append_zero_is_noop(void) +{ + lf_buf b; + + lf_buf_init(&b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_buf_append(&b, NULL, 0)); + TEST_ASSERT_EQUAL_UINT(0, b.len); + lf_buf_free(&b); +} + +static void test_clear_keeps_capacity(void) +{ + lf_buf b; + size_t cap; + + lf_buf_init(&b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_buf_append(&b, "hello", 5)); + cap = b.cap; + lf_buf_clear(&b); + TEST_ASSERT_EQUAL_UINT(0, b.len); + TEST_ASSERT_EQUAL_UINT(cap, b.cap); + lf_buf_free(&b); +} + +static void test_reserve_never_shrinks(void) +{ + lf_buf b; + size_t cap; + + lf_buf_init(&b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_buf_reserve(&b, 100)); + cap = b.cap; + TEST_ASSERT_TRUE(cap >= 100); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_buf_reserve(&b, 10)); + TEST_ASSERT_EQUAL_UINT(cap, b.cap); + lf_buf_free(&b); +} + +static void test_u16le_layout(void) +{ + lf_buf b; + + lf_buf_init(&b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_buf_put_u16le(&b, 0x1234)); + TEST_ASSERT_EQUAL_UINT(2, b.len); + TEST_ASSERT_EQUAL_UINT(0x34, b.data[0]); + TEST_ASSERT_EQUAL_UINT(0x12, b.data[1]); + lf_buf_free(&b); +} + +static void test_u32le_layout(void) +{ + lf_buf b; + + lf_buf_init(&b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_buf_put_u32le(&b, 0x12345678u)); + TEST_ASSERT_EQUAL_UINT(4, b.len); + TEST_ASSERT_EQUAL_UINT(0x78, b.data[0]); + TEST_ASSERT_EQUAL_UINT(0x56, b.data[1]); + TEST_ASSERT_EQUAL_UINT(0x34, b.data[2]); + TEST_ASSERT_EQUAL_UINT(0x12, b.data[3]); + lf_buf_free(&b); +} + +static void test_u64le_layout(void) +{ + lf_buf b; + + lf_buf_init(&b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_buf_put_u64le(&b, 0x0102030405060708ull)); + TEST_ASSERT_EQUAL_UINT(8, b.len); + TEST_ASSERT_EQUAL_UINT(0x08, b.data[0]); + TEST_ASSERT_EQUAL_UINT(0x07, b.data[1]); + TEST_ASSERT_EQUAL_UINT(0x06, b.data[2]); + TEST_ASSERT_EQUAL_UINT(0x05, b.data[3]); + TEST_ASSERT_EQUAL_UINT(0x01, b.data[7]); + lf_buf_free(&b); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_init_is_empty); + RUN_TEST(test_append_grows); + RUN_TEST(test_append_zero_is_noop); + RUN_TEST(test_clear_keeps_capacity); + RUN_TEST(test_reserve_never_shrinks); + RUN_TEST(test_u16le_layout); + RUN_TEST(test_u32le_layout); + RUN_TEST(test_u64le_layout); + return UNITY_END(); +} diff --git a/tests/test_huffman.c b/tests/test_huffman.c new file mode 100644 index 0000000..cc6b363 --- /dev/null +++ b/tests/test_huffman.c @@ -0,0 +1,241 @@ +#include "unity.h" + +#include "lf_bitio.h" +#include "lf_buf.h" +#include "lf_huffman.h" + +#include + +void setUp(void) +{ +} + +void tearDown(void) +{ +} + +static uint32_t rng_state = 1u; + +static uint32_t next_rng(void) +{ + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; + return rng_state; +} + +static void check_code_valid(const uint8_t *lengths, const uint32_t *codes, + uint32_t n) +{ + uint32_t i; + uint32_t j; + uint64_t kraft = 0; + + for (i = 0; i < n; i++) { + if (lengths[i] == 0) { + continue; + } + TEST_ASSERT_TRUE(lengths[i] <= LF_MAX_CODE_LEN); + kraft += (uint64_t)1 << (LF_MAX_CODE_LEN - lengths[i]); + for (j = 0; j < n; j++) { + if (j != i && lengths[j] != 0 && lengths[j] <= lengths[i]) { + unsigned shift = lengths[i] - lengths[j]; + + TEST_ASSERT_NOT_EQUAL_UINT(codes[i] >> shift, codes[j]); + } + } + } + TEST_ASSERT_TRUE(kraft <= ((uint64_t)1 << LF_MAX_CODE_LEN)); +} + +static void test_single_symbol(void) +{ + uint32_t freq[LF_NUM_LITLEN] = {0}; + uint8_t lengths[LF_NUM_LITLEN]; + uint32_t codes[LF_NUM_LITLEN]; + + freq[100] = 7; + memset(lengths, 0, sizeof(lengths)); + lf_huff_lengths(lengths, freq, LF_NUM_LITLEN); + TEST_ASSERT_EQUAL_UINT(1, lengths[100]); + TEST_ASSERT_EQUAL_UINT(0, lengths[99]); + lf_huff_canonical(codes, lengths, LF_NUM_LITLEN); + TEST_ASSERT_EQUAL_UINT(0, codes[100]); +} + +static void test_no_symbols(void) +{ + uint32_t freq[LF_NUM_LITLEN] = {0}; + uint8_t lengths[LF_NUM_LITLEN]; + + memset(lengths, 0xFF, sizeof(lengths)); + lf_huff_lengths(lengths, freq, LF_NUM_LITLEN); + TEST_ASSERT_EQUAL_UINT(0, lengths[0]); + TEST_ASSERT_EQUAL_UINT(0, lengths[LF_NUM_LITLEN - 1]); +} + +static void test_balanced_tree(void) +{ + uint32_t freq[16]; + uint8_t lengths[16]; + uint32_t codes[16]; + unsigned i; + + for (i = 0; i < 16; i++) { + freq[i] = 1; + } + lf_huff_lengths(lengths, freq, 16); + for (i = 0; i < 16; i++) { + TEST_ASSERT_EQUAL_UINT(4, lengths[i]); + } + lf_huff_canonical(codes, lengths, 16); + check_code_valid(lengths, codes, 16); +} + +static void test_length_cap_many_symbols(void) +{ + uint32_t freq[300]; + uint8_t lengths[300]; + uint32_t codes[300]; + unsigned i; + + for (i = 0; i < 300; i++) { + freq[i] = 1; + } + lf_huff_lengths(lengths, freq, 300); + for (i = 0; i < 300; i++) { + TEST_ASSERT_TRUE(lengths[i] > 0); + TEST_ASSERT_TRUE(lengths[i] <= LF_MAX_CODE_LEN); + } + lf_huff_canonical(codes, lengths, 300); + check_code_valid(lengths, codes, 300); +} + +static void test_random_distributions(void) +{ + uint32_t freq[200]; + uint8_t lengths[200]; + uint32_t codes[200]; + unsigned iter; + unsigned i; + + for (iter = 0; iter < 20; iter++) { + for (i = 0; i < 200; i++) { + freq[i] = next_rng() % 100u; + } + lf_huff_lengths(lengths, freq, 200); + for (i = 0; i < 200; i++) { + TEST_ASSERT_TRUE(lengths[i] <= LF_MAX_CODE_LEN); + if (freq[i] == 0) { + TEST_ASSERT_EQUAL_UINT(0, lengths[i]); + } + } + lf_huff_canonical(codes, lengths, 200); + check_code_valid(lengths, codes, 200); + } +} + +static void test_decode_roundtrip(void) +{ + const uint32_t n = 100; + uint32_t freq[100]; + uint8_t lengths[100]; + uint32_t codes[100]; + uint32_t chosen[1000]; + lf_huff_dec dec; + lf_buf b; + lf_bwriter w; + lf_breader r; + unsigned i; + + memset(freq, 0, sizeof(freq)); + for (i = 0; i < 1000; i++) { + chosen[i] = next_rng() % n; + freq[chosen[i]]++; + } + + lf_huff_lengths(lengths, freq, n); + lf_huff_canonical(codes, lengths, n); + lf_huff_dec_init(&dec, lengths, n); + TEST_ASSERT_TRUE(dec.valid); + + lf_buf_init(&b); + lf_bwriter_init(&w, &b); + for (i = 0; i < 1000; i++) { + TEST_ASSERT_EQUAL_INT(LF_OK, + lf_bwriter_put_code(&w, codes[chosen[i]], + lengths[chosen[i]])); + } + lf_bwriter_align(&w); + + lf_breader_init(&r, b.data, b.len); + for (i = 0; i < 1000; i++) { + int sym = lf_huff_dec_symbol(&dec, &r); + + TEST_ASSERT_TRUE(sym >= 0); + TEST_ASSERT_EQUAL_INT((int)chosen[i], sym); + } + lf_buf_free(&b); +} + +static void test_decode_consumes_whole_stream(void) +{ + uint8_t lengths[4] = {1, 2, 3, 0}; + uint32_t codes[4]; + lf_huff_dec dec; + lf_buf b; + lf_bwriter w; + lf_breader r; + int count; + + lf_huff_canonical(codes, lengths, 4); + lf_huff_dec_init(&dec, lengths, 4); + + lf_buf_init(&b); + lf_bwriter_init(&w, &b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_put_code(&w, codes[2], 3)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_bwriter_align(&w)); + + lf_breader_init(&r, b.data, b.len); + for (count = 0; count < 100; count++) { + if (lf_huff_dec_symbol(&dec, &r) < 0) { + break; + } + } + TEST_ASSERT_TRUE(count < 100); + TEST_ASSERT_FALSE(lf_breader_ok(&r)); + lf_buf_free(&b); +} + +static void test_decode_fails_when_input_runs_out(void) +{ + uint8_t lengths[4] = {1, 2, 3, 0}; + lf_huff_dec dec; + uint8_t data[1] = {0x00}; + lf_breader r; + int calls; + + lf_huff_dec_init(&dec, lengths, 4); + lf_breader_init(&r, data, sizeof(data)); + for (calls = 0; calls < 64; calls++) { + if (lf_huff_dec_symbol(&dec, &r) < 0) { + break; + } + } + TEST_ASSERT_TRUE(calls < 64); + TEST_ASSERT_FALSE(lf_breader_ok(&r)); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_single_symbol); + RUN_TEST(test_no_symbols); + RUN_TEST(test_balanced_tree); + RUN_TEST(test_length_cap_many_symbols); + RUN_TEST(test_random_distributions); + RUN_TEST(test_decode_roundtrip); + RUN_TEST(test_decode_consumes_whole_stream); + RUN_TEST(test_decode_fails_when_input_runs_out); + return UNITY_END(); +} diff --git a/tests/test_match.c b/tests/test_match.c new file mode 100644 index 0000000..ca76d9f --- /dev/null +++ b/tests/test_match.c @@ -0,0 +1,132 @@ +#include "unity.h" + +#include "lf_match.h" + +#include + +void setUp(void) +{ +} + +void tearDown(void) +{ +} + +static void test_init_rejects_bad_params(void) +{ + lf_match m; + + TEST_ASSERT_EQUAL_INT(LF_ERR_PARAM, lf_match_init(&m, 0, 16)); + TEST_ASSERT_EQUAL_INT(LF_ERR_PARAM, lf_match_init(&m, 8, 0)); + TEST_ASSERT_EQUAL_INT(LF_ERR_PARAM, lf_match_init(&m, 1, 16)); + TEST_ASSERT_EQUAL_INT(LF_ERR_PARAM, lf_match_init(NULL, 32, 16)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_match_init(&m, 32, 16)); + lf_match_free(&m); +} + +static void test_finds_obvious_match(void) +{ + const uint8_t data[] = "abcabcabcabc"; + lf_match m; + unsigned ml = 0; + uint32_t md = 0; + size_t i; + + TEST_ASSERT_EQUAL_INT(LF_OK, lf_match_init(&m, 32, 64)); + + for (i = 0; i < 3; i++) { + lf_match_insert(&m, data, i, sizeof(data) - 1); + } + TEST_ASSERT_EQUAL_INT(LF_OK, + lf_match_search(&m, data, 3, sizeof(data) - 1, &ml, &md)); + TEST_ASSERT_TRUE(ml >= 3); + TEST_ASSERT_EQUAL_UINT(3, md); + TEST_ASSERT_TRUE(ml + 3 <= sizeof(data) - 1); + lf_match_free(&m); +} + +static void test_no_match_at_start(void) +{ + const uint8_t data[] = "abcdefghij"; + lf_match m; + unsigned ml = 99; + uint32_t md = 99; + + TEST_ASSERT_EQUAL_INT(LF_OK, lf_match_init(&m, 32, 64)); + TEST_ASSERT_EQUAL_INT(LF_OK, + lf_match_search(&m, data, 0, sizeof(data) - 1, &ml, &md)); + TEST_ASSERT_EQUAL_UINT(0, ml); + TEST_ASSERT_EQUAL_UINT(0, md); + lf_match_free(&m); +} + +static void test_window_limits_distance(void) +{ + static uint8_t data[600]; + lf_match m; + unsigned ml = 0; + uint32_t md = 0; + + memset(data, 0xAB, sizeof(data)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_match_init(&m, 64, 4096)); + + { + size_t i; + + for (i = 0; i < 128; i++) { + lf_match_insert(&m, data, i, sizeof(data)); + } + } + TEST_ASSERT_EQUAL_INT(LF_OK, + lf_match_search(&m, data, 128, sizeof(data), &ml, &md)); + TEST_ASSERT_TRUE(ml >= 3); + TEST_ASSERT_TRUE(md <= 64); + lf_match_free(&m); +} + +static void test_reset_clears_history(void) +{ + const uint8_t data[] = "aaaaaa"; + lf_match m; + unsigned ml = 0; + uint32_t md = 0; + + TEST_ASSERT_EQUAL_INT(LF_OK, lf_match_init(&m, 32, 64)); + lf_match_insert(&m, data, 0, sizeof(data) - 1); + lf_match_reset(&m); + TEST_ASSERT_EQUAL_INT(LF_OK, + lf_match_search(&m, data, 1, sizeof(data) - 1, &ml, &md)); + TEST_ASSERT_EQUAL_UINT(0, ml); + lf_match_free(&m); +} + +static void test_overlapping_match(void) +{ + const uint8_t data[] = "abcabcabcab"; + lf_match m; + unsigned ml = 0; + uint32_t md = 0; + size_t i; + + TEST_ASSERT_EQUAL_INT(LF_OK, lf_match_init(&m, 32, 64)); + for (i = 0; i < 3; i++) { + lf_match_insert(&m, data, i, sizeof(data) - 1); + } + TEST_ASSERT_EQUAL_INT(LF_OK, + lf_match_search(&m, data, 3, sizeof(data) - 1, &ml, &md)); + TEST_ASSERT_EQUAL_UINT(3, md); + TEST_ASSERT_TRUE(ml >= 3); + lf_match_free(&m); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_init_rejects_bad_params); + RUN_TEST(test_finds_obvious_match); + RUN_TEST(test_no_match_at_start); + RUN_TEST(test_window_limits_distance); + RUN_TEST(test_reset_clears_history); + RUN_TEST(test_overlapping_match); + return UNITY_END(); +} diff --git a/tests/test_stream.c b/tests/test_stream.c new file mode 100644 index 0000000..daed187 --- /dev/null +++ b/tests/test_stream.c @@ -0,0 +1,486 @@ +#include "unity.h" + +#include "lf_buf.h" +#include "lf_common.h" +#include "lf_stream.h" + +#include +#include + +void setUp(void) +{ +} + +void tearDown(void) +{ +} + +static uint32_t rng_state = 12345u; + +static void rng_seed(uint32_t seed) +{ + rng_state = seed; +} + +static uint32_t next_rng(void) +{ + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; + return rng_state; +} + +static void roundtrip(const uint8_t *data, size_t len) +{ + lf_buf packed; + lf_buf plain; + lf_result r; + + lf_buf_init(&packed); + lf_buf_init(&plain); + r = lf_compress(NULL, data, len, &packed); + TEST_ASSERT_EQUAL_INT(LF_OK, r); + TEST_ASSERT_TRUE(packed.len <= lf_compress_bound(len)); + + r = lf_decompress(packed.data, packed.len, &plain); + TEST_ASSERT_EQUAL_INT(LF_OK, r); + TEST_ASSERT_EQUAL_UINT(len, plain.len); + if (len > 0) { + TEST_ASSERT_EQUAL_INT(0, memcmp(plain.data, data, len)); + } + lf_buf_free(&packed); + lf_buf_free(&plain); +} + +static void test_roundtrip_empty(void) +{ + roundtrip(NULL, 0); +} + +static void test_roundtrip_single_byte(void) +{ + uint8_t d[1] = {0x7F}; + + roundtrip(d, 1); +} + +static void test_roundtrip_two_bytes(void) +{ + uint8_t d[2] = {0x00, 0xFF}; + + roundtrip(d, 2); +} + +static void test_roundtrip_text(void) +{ + const char *text = + "leaflitter compresses and decompresses byte streams. It uses a match " + "finder and Huffman coding, similar to DEFLATE."; + + roundtrip((const uint8_t *)text, strlen(text)); +} + +static void test_roundtrip_repeated_text(void) +{ + char text[8192]; + size_t i; + + for (i = 0; i < sizeof(text); i++) { + text[i] = (char)("leaflitter byte stream\n"[i % 24]); + } + roundtrip((const uint8_t *)text, sizeof(text)); +} + +static void test_roundtrip_random(void) +{ + uint8_t data[20000]; + size_t i; + + rng_seed(99); + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)next_rng(); + } + roundtrip(data, sizeof(data)); +} + +static void test_roundtrip_large_multi_block(void) +{ + uint8_t *data; + size_t n = 300000; + size_t i; + + data = (uint8_t *)malloc(n); + TEST_ASSERT_NOT_NULL(data); + rng_seed(7); + for (i = 0; i < n; i++) { + data[i] = (uint8_t)("compression pattern %d"[i % 22]); + } + for (i = 100000; i < 120000; i++) { + data[i] = (uint8_t)next_rng(); + } + roundtrip(data, n); + free(data); +} + +static void test_all_byte_values(void) +{ + uint8_t data[256]; + size_t i; + + for (i = 0; i < 256; i++) { + data[i] = (uint8_t)i; + } + roundtrip(data, sizeof(data)); +} + +static void test_compress_is_deterministic(void) +{ + uint8_t data[5000]; + lf_buf a; + lf_buf b; + size_t i; + + rng_seed(5); + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)next_rng(); + } + for (i = 0; i < sizeof(data); i += 37) { + data[i] = (uint8_t)(i % 256); + } + + lf_buf_init(&a); + lf_buf_init(&b); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_compress(NULL, data, sizeof(data), &a)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_compress(NULL, data, sizeof(data), &b)); + TEST_ASSERT_EQUAL_UINT(a.len, b.len); + TEST_ASSERT_EQUAL_INT(0, memcmp(a.data, b.data, a.len)); + lf_buf_free(&a); + lf_buf_free(&b); +} + +static void test_compress_bound_monotonic(void) +{ + TEST_ASSERT_TRUE(lf_compress_bound(0) > 0); + TEST_ASSERT_TRUE(lf_compress_bound(100) <= lf_compress_bound(200)); + TEST_ASSERT_TRUE(lf_compress_bound(10000) > 10000); +} + +static void test_rejects_bad_magic(void) +{ + uint8_t data[40]; + lf_buf out; + + memset(data, 0, sizeof(data)); + data[0] = 'X'; + data[1] = 'F'; + data[2] = 0x01; + data[3] = 1; + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_ERR_BAD_MAGIC, lf_decompress(data, sizeof(data), &out)); + lf_buf_free(&out); +} + +static void test_rejects_bad_version(void) +{ + uint8_t data[40]; + lf_buf out; + + memset(data, 0, sizeof(data)); + data[0] = LF_MAGIC_0; + data[1] = LF_MAGIC_1; + data[2] = LF_MAGIC_2; + data[3] = 99; + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_ERR_UNSUPPORTED_VERSION, + lf_decompress(data, sizeof(data), &out)); + lf_buf_free(&out); +} + +static void test_rejects_truncated_header(void) +{ + uint8_t data[10] = {0}; + lf_buf out; + + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_ERR_TRUNCATED, lf_decompress(data, sizeof(data), &out)); + lf_buf_free(&out); +} + +static void test_rejects_truncated_body(void) +{ + uint8_t data[1000]; + lf_buf packed; + lf_buf out; + size_t i; + + rng_seed(3); + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)next_rng(); + } + lf_buf_init(&packed); + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_compress(NULL, data, sizeof(data), &packed)); + TEST_ASSERT_TRUE(packed.len > 24); + TEST_ASSERT_NOT_EQUAL_INT(LF_OK, + lf_decompress(packed.data, packed.len - 1, &out)); + lf_buf_free(&packed); + lf_buf_free(&out); +} + +static void test_rejects_corrupted_adler(void) +{ + uint8_t data[500]; + lf_buf packed; + lf_buf out; + size_t i; + + rng_seed(11); + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)next_rng(); + } + lf_buf_init(&packed); + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_compress(NULL, data, sizeof(data), &packed)); + packed.data[packed.len / 2] ^= 0xFF; + TEST_ASSERT_NOT_EQUAL_INT(LF_OK, lf_decompress(packed.data, packed.len, &out)); + lf_buf_free(&packed); + lf_buf_free(&out); +} + +static void test_rejects_trailing_garbage(void) +{ + uint8_t data[64]; + lf_buf packed; + lf_buf out; + size_t i; + + rng_seed(21); + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)(i % 4u); + } + lf_buf_init(&packed); + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_compress(NULL, data, sizeof(data), &packed)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_buf_append(&packed, "tail", 4)); + TEST_ASSERT_EQUAL_INT(LF_ERR_INVALID_DATA, + lf_decompress(packed.data, packed.len, &out)); + lf_buf_free(&packed); + lf_buf_free(&out); +} + +static void test_rejects_null_params(void) +{ + lf_buf out; + + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_ERR_PARAM, lf_compress(NULL, NULL, 5, &out)); + TEST_ASSERT_EQUAL_INT(LF_ERR_PARAM, lf_compress(NULL, NULL, 0, NULL)); + TEST_ASSERT_EQUAL_INT(LF_ERR_PARAM, lf_decompress(NULL, 24, &out)); + TEST_ASSERT_EQUAL_INT(LF_ERR_PARAM, lf_decompress("x", 1, NULL)); + lf_buf_free(&out); +} + +static void test_small_window_and_block_options(void) +{ + uint8_t data[20000]; + lf_options opt; + lf_buf packed; + lf_buf plain; + size_t i; + + rng_seed(8); + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)("repeating option test"[i % 21]); + } + + lf_options_default(&opt); + opt.window_size = 1024; + opt.block_size = 4096; + opt.max_chain = 64; + + lf_buf_init(&packed); + lf_buf_init(&plain); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_compress(&opt, data, sizeof(data), &packed)); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_decompress(packed.data, packed.len, &plain)); + TEST_ASSERT_EQUAL_UINT(sizeof(data), plain.len); + TEST_ASSERT_EQUAL_INT(0, memcmp(plain.data, data, sizeof(data))); + lf_buf_free(&packed); + lf_buf_free(&plain); +} + +static void test_roundtrip_huge_single_block(void) +{ + uint8_t data[200000]; + size_t i; + + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)("abcdefghij"[i % 10]); + } + roundtrip(data, sizeof(data)); +} + +static const uint8_t golden_empty[] = { + 0x4c, 0x46, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const uint8_t golden_abc[] = { + 0x4c, 0x46, 0x01, 0x01, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x27, 0x01, 0x4d, 0x02, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x03, 0x00, 0x00, 0x00, 0x61, 0x62, 0x63, +}; + +static const uint8_t golden_text[] = { + 0x4c, 0x46, 0x01, 0x01, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x21, 0x2c, 0xb2, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x0d, 0x01, 0x08, 0x00, 0x4a, 0xe1, 0x05, 0xa2, 0x9f, 0x4d, 0x34, + 0xac, 0x69, 0x55, 0xa5, 0x56, 0x74, 0xad, 0x34, 0xac, 0xe3, 0x01, 0xd1, + 0xf7, 0xa2, 0x03, 0x42, 0x5a, 0x0d, 0x71, 0x19, 0x6c, 0xc8, 0xbe, 0xf5, + 0xc9, 0x29, 0x14, 0x2d, 0xd6, 0x55, 0x07, 0xd9, 0xbc, +}; + +static void check_golden(const uint8_t *input, size_t in_len, + const uint8_t *expect, size_t expect_len) +{ + lf_buf packed; + + lf_buf_init(&packed); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_compress(NULL, input, in_len, &packed)); + TEST_ASSERT_EQUAL_UINT(expect_len, packed.len); + TEST_ASSERT_EQUAL_INT(0, memcmp(packed.data, expect, expect_len)); + lf_buf_free(&packed); +} + +static void test_golden_vectors(void) +{ + const char *text = + "leaflitter compresses bytes. leaflitter compresses bytes. " + "leaflitter compresses bytes."; + + check_golden(NULL, 0, golden_empty, sizeof(golden_empty)); + check_golden((const uint8_t *)"abc", 3, golden_abc, sizeof(golden_abc)); + check_golden((const uint8_t *)text, strlen(text), golden_text, + sizeof(golden_text)); +} + +static void test_block_boundary_sizes(void) +{ + static const size_t sizes[] = {65535, 65536, 65537, 131071, 131072, 131073}; + uint8_t *data; + size_t idx; + size_t i; + + data = (uint8_t *)malloc(sizes[sizeof(sizes) / sizeof(sizes[0]) - 1]); + TEST_ASSERT_NOT_NULL(data); + for (i = 0; i < sizes[sizeof(sizes) / sizeof(sizes[0]) - 1]; i++) { + data[i] = (uint8_t)("boundary test"[i % 13]); + } + for (idx = 0; idx < sizeof(sizes) / sizeof(sizes[0]); idx++) { + roundtrip(data, sizes[idx]); + } + free(data); +} + +static void test_highly_repetitive_input(void) +{ + uint8_t data[100000]; + size_t i; + + for (i = 0; i < sizeof(data); i++) { + data[i] = 0x55; + } + roundtrip(data, sizeof(data)); +} + +static void test_small_alphabet_random(void) +{ + uint8_t data[40000]; + size_t i; + + rng_seed(17); + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)(next_rng() % 4u); + } + roundtrip(data, sizeof(data)); +} + +static void test_decode_never_crashes_on_garbage(void) +{ + uint8_t buf[64]; + lf_buf out; + unsigned iter; + unsigned len; + + lf_buf_init(&out); + for (iter = 0; iter < 2000; iter++) { + len = next_rng() % sizeof(buf); + rng_seed(iter + 1); + while (1) { + size_t i; + + for (i = 0; i < len; i++) { + buf[i] = (uint8_t)next_rng(); + } + break; + } + (void)lf_decompress(buf, len, &out); + lf_buf_clear(&out); + } + lf_buf_free(&out); +} + +static void test_corrupt_every_byte(void) +{ + uint8_t data[3000]; + lf_buf packed; + lf_buf out; + size_t i; + size_t j; + + rng_seed(29); + for (i = 0; i < sizeof(data); i++) { + data[i] = (uint8_t)next_rng(); + } + lf_buf_init(&packed); + lf_buf_init(&out); + TEST_ASSERT_EQUAL_INT(LF_OK, lf_compress(NULL, data, sizeof(data), &packed)); + for (j = 24; j < packed.len; j++) { + packed.data[j] ^= 0x40; + (void)lf_decompress(packed.data, packed.len, &out); + lf_buf_clear(&out); + packed.data[j] ^= 0x40; + } + lf_buf_free(&packed); + lf_buf_free(&out); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_roundtrip_empty); + RUN_TEST(test_roundtrip_single_byte); + RUN_TEST(test_roundtrip_two_bytes); + RUN_TEST(test_roundtrip_text); + RUN_TEST(test_roundtrip_repeated_text); + RUN_TEST(test_roundtrip_random); + RUN_TEST(test_roundtrip_large_multi_block); + RUN_TEST(test_all_byte_values); + RUN_TEST(test_compress_is_deterministic); + RUN_TEST(test_compress_bound_monotonic); + RUN_TEST(test_rejects_bad_magic); + RUN_TEST(test_rejects_bad_version); + RUN_TEST(test_rejects_truncated_header); + RUN_TEST(test_rejects_truncated_body); + RUN_TEST(test_rejects_corrupted_adler); + RUN_TEST(test_rejects_trailing_garbage); + RUN_TEST(test_rejects_null_params); + RUN_TEST(test_small_window_and_block_options); + RUN_TEST(test_roundtrip_huge_single_block); + RUN_TEST(test_golden_vectors); + RUN_TEST(test_block_boundary_sizes); + RUN_TEST(test_highly_repetitive_input); + RUN_TEST(test_small_alphabet_random); + RUN_TEST(test_decode_never_crashes_on_garbage); + RUN_TEST(test_corrupt_every_byte); + return UNITY_END(); +} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000..3f23dca --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,3 @@ +add_executable(leaflitter_cli leaflitter.c) +target_link_libraries(leaflitter_cli PRIVATE leaflitter::leaflitter) +set_target_properties(leaflitter_cli PROPERTIES OUTPUT_NAME leaflitter) diff --git a/tools/leaflitter.c b/tools/leaflitter.c new file mode 100644 index 0000000..3b5e7eb --- /dev/null +++ b/tools/leaflitter.c @@ -0,0 +1,202 @@ +#include "lf_common.h" +#include "lf_stream.h" + +#include +#include +#include + +#define LF_TOOL_VERSION "1.1.0" + +static void lf_tool_usage(FILE *f) +{ + fprintf(f, + "leaflitter %s\n" + "Usage: leaflitter \n" + "Commands:\n" + " c Compress to .\n" + " d Decompress to .\n" + " i Show the header of .\n", + LF_TOOL_VERSION); +} + +static void *lf_tool_read_file(const char *path, size_t *out_len) +{ + FILE *fp; + long size; + uint8_t *data; + + fp = fopen(path, "rb"); + if (fp == NULL) { + fprintf(stderr, "leaflitter: cannot open %s\n", path); + return NULL; + } + if (fseek(fp, 0, SEEK_END) != 0 || (size = ftell(fp)) < 0 || + fseek(fp, 0, SEEK_SET) != 0) { + fprintf(stderr, "leaflitter: cannot size %s\n", path); + fclose(fp); + return NULL; + } + data = (uint8_t *)malloc((size_t)size + 1); + if (data == NULL) { + fprintf(stderr, "leaflitter: out of memory\n"); + fclose(fp); + return NULL; + } + if (size > 0 && fread(data, 1, (size_t)size, fp) != (size_t)size) { + fprintf(stderr, "leaflitter: cannot read %s\n", path); + free(data); + fclose(fp); + return NULL; + } + fclose(fp); + *out_len = (size_t)size; + return data; +} + +static int lf_tool_write_file(const char *path, const void *data, size_t len) +{ + FILE *fp; + int ok; + + fp = fopen(path, "wb"); + if (fp == NULL) { + fprintf(stderr, "leaflitter: cannot open %s\n", path); + return 0; + } + ok = (len == 0 || fwrite(data, 1, len, fp) == len); + if (fclose(fp) != 0) { + ok = 0; + } + if (!ok) { + fprintf(stderr, "leaflitter: cannot write %s\n", path); + } + return ok; +} + +static int lf_tool_compress(const char *in, const char *out) +{ + void *data; + size_t len; + lf_buf c; + lf_result r; + int ok = 0; + + data = lf_tool_read_file(in, &len); + if (data == NULL) { + return 1; + } + lf_buf_init(&c); + r = lf_compress(NULL, data, len, &c); + if (r != LF_OK) { + fprintf(stderr, "leaflitter: compress failed: %s\n", lf_result_name(r)); + } else if (!lf_tool_write_file(out, c.data, c.len)) { + fprintf(stderr, "leaflitter: cannot write %s\n", out); + } else { + printf("%s: %zu -> %zu bytes\n", in, len, c.len); + ok = 1; + } + lf_buf_free(&c); + free(data); + return ok ? 0 : 1; +} + +static int lf_tool_decompress(const char *in, const char *out) +{ + void *data; + size_t len; + lf_buf p; + lf_result r; + int ok = 0; + + data = lf_tool_read_file(in, &len); + if (data == NULL) { + return 1; + } + lf_buf_init(&p); + r = lf_decompress(data, len, &p); + if (r != LF_OK) { + fprintf(stderr, "leaflitter: decompress failed: %s\n", lf_result_name(r)); + } else if (!lf_tool_write_file(out, p.data, p.len)) { + fprintf(stderr, "leaflitter: cannot write %s\n", out); + } else { + printf("%s: %zu -> %zu bytes\n", in, len, p.len); + ok = 1; + } + lf_buf_free(&p); + free(data); + return ok ? 0 : 1; +} + +static int lf_tool_info(const char *in) +{ + void *data; + size_t len; + const uint8_t *h; + uint64_t size; + uint32_t adler; + uint32_t window; + int ok = 0; + + data = lf_tool_read_file(in, &len); + if (data == NULL) { + return 1; + } + if (len < 24) { + fprintf(stderr, "leaflitter: %s is too short to be a stream\n", in); + free(data); + return 1; + } + h = (const uint8_t *)data; + if (h[0] != LF_MAGIC_0 || h[1] != LF_MAGIC_1 || h[2] != LF_MAGIC_2) { + fprintf(stderr, "leaflitter: %s has no leaflitter magic\n", in); + free(data); + return 1; + } + size = (uint64_t)h[4] | ((uint64_t)h[5] << 8) | ((uint64_t)h[6] << 16) | + ((uint64_t)h[7] << 24) | ((uint64_t)h[8] << 32) | + ((uint64_t)h[9] << 40) | ((uint64_t)h[10] << 48) | + ((uint64_t)h[11] << 56); + adler = (uint32_t)h[12] | ((uint32_t)h[13] << 8) | + ((uint32_t)h[14] << 16) | ((uint32_t)h[15] << 24); + window = (uint32_t)h[16] | ((uint32_t)h[17] << 8) | + ((uint32_t)h[18] << 16) | ((uint32_t)h[19] << 24); + printf("format version: %u\n", h[3]); + printf("uncompressed size: %llu bytes\n", (unsigned long long)size); + printf("adler32: %08x\n", adler); + printf("window size: %u bytes\n", window); + ok = 1; + free(data); + return ok ? 0 : 1; +} + +int main(int argc, char **argv) +{ + if (argc < 3 || argc > 4) { + lf_tool_usage(stderr); + return 2; + } + if (strcmp(argv[1], "c") == 0) { + if (argc != 4) { + lf_tool_usage(stderr); + return 2; + } + return lf_tool_compress(argv[2], argv[3]); + } + if (strcmp(argv[1], "d") == 0) { + if (argc != 4) { + lf_tool_usage(stderr); + return 2; + } + return lf_tool_decompress(argv[2], argv[3]); + } + if (strcmp(argv[1], "i") == 0) { + if (argc != 3) { + lf_tool_usage(stderr); + return 2; + } + return lf_tool_info(argv[2]); + } + fprintf(stderr, "leaflitter: unknown command %s\n", argv[1]); + lf_tool_usage(stderr); + return 2; +}