Add safe high-performance decoding and CLI

.github/workflows/build_and_test.yaml

@@ -17,50 +17,22 @@
 
 jobs:
   build_and_test:
-    strategy:
-      matrix:
-        include:
-          - name: Ubuntu 20.04 - Release
-            runner: ubuntu-20.04
-            cargo_flags: --release
-
-          - name: Ubuntu 20.04 - Debug
-            runner: ubuntu-20.04
-            cargo_flags:
-
-    name: ${{ matrix.name }}
-    runs-on: ${{ matrix.runner }}
+    name: Ubuntu 24.04
+    runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v3
-    - uses: Swatinem/rust-cache@v2
-      with:
-        # rust-cache already handles all the sane defaults for caching rust builds.
-        # However, because we are running separate debug/release builds in parallel,
-        # we also need to add the runner and cargo_flags to the key so that a separate cache is used.
-        # Otherwise, only the last build to finish would get saved to the cache.
-        key: ${{ matrix.runner }} - ${{ matrix.cargo_flags }}
-    - name: Install external deps for the prom-remote-api crate
-      run: sudo apt-get install protobuf-compiler
-    - name: Install cargo-hack
-      run: cargo install cargo-hack --version 0.5.8
-    - name: Check `cargo fmt` was run
-      run: cargo fmt --all -- --check
-    - name: Ensure that all crates compile and have no warnings under every possible combination of features
-      # some things to explicitly point out:
-      # * clippy also reports rustc warnings and errors
-      # * clippy --all-targets causes clippy to run against tests and examples which it doesn't do by default.
-      run: |
-        # Display all clippy lint failures as warnings
-        cargo hack --feature-powerset clippy --all-targets --locked ${{ matrix.cargo_flags }}
-        # Fail CI on the first crate to fail clippy lints
-        cargo hack --feature-powerset clippy --all-targets --locked ${{ matrix.cargo_flags }} -- -D warnings
-    - name: Ensure that tests pass
-      run: |
-        cargo test --doc ${{ matrix.cargo_flags }} --all-features -- --show-output --nocapture
-        cargo test ${{ matrix.cargo_flags }} --all-features --all-targets -- --nocapture
-    - name: Ensure that tests did not create or modify any files that arent .gitignore'd
-      run: |
-        if [ -n "$(git status --porcelain)" ]; then
-          git status
-          exit 1
-        fi
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+      - name: Check `cargo fmt` was run
+        run: cargo fmt --all -- --check
+      - name: Ensure that all targets compile without warnings
+        run: cargo clippy --workspace --all-targets --locked -- -D warnings
+      - name: Ensure that all tests pass
+        run: cargo test --workspace --all-targets --locked
+      - name: Ensure that the decompression benchmark compiles
+        run: cargo bench -p atsc --bench decompression_bench --no-run
+      - name: Ensure that tests did not create or modify any files that arent .gitignore'd
+        run: |
+          if [ -n "$(git status --porcelain)" ]; then
+            git status
+            exit 1
+          fi

.github/workflows/release.yaml

@@ -9,22 +9,22 @@ on:
 jobs:
   prepublish-check:
     name: "Check that the project is releaseable"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
       - name: Run checks
         run: atsc/release/is_releasable.sh
 
   publish-binary:
     name: "Publish Binary to GitHub"
     needs: prepublish-check
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
       - name: Build & test
         run: atsc/release/build_release.sh
       - name: Publish
-        uses: marvinpinto/action-automatic-releases@latest
+        uses: marvinpinto/action-automatic-releases@d68defdd11f9dcc7f52f35c1b7c236ee7513bcc1
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           prerelease: false

Cargo.toml

@@ -14,6 +14,7 @@ opt-level = 0
 [profile.release]
 opt-level = 3
 codegen-units = 1
+lto = "thin"
 
 [workspace.dependencies]
 clap = {version = "4.3.14", features = ["derive"] }

README.md

@@ -23,7 +23,7 @@ The fastest way to test ATSC is with a CSV file!
 3. Execute the following command:
 
     ```bash
-    cargo run --release -- --csv <input-file>
+    cargo run --release -- compress --csv <input-file>
     ```
 
 4. You have a compressed timeseries!
@@ -91,70 +91,74 @@ For full documentation please go to [Docs](https://github.com/instaclustr/atsc/t
 
 ### Prerequisites
 
-* Ensure you have [Rust](https://www.rust-lang.org/tools/install) installed on your system.
+* Ensure you have [Rust 1.81 or newer](https://www.rust-lang.org/tools/install) installed on your system.
 
 ### Usage
 
-ATSC relies on files with a WBRO extension to operate, learn more about that here: [WBRO - A time series format](https://github.com/instaclustr/atsc/tree/main/wavbrro)
-You can also compress from CSV with the provided [CSV tool](https://github.com/instaclustr/atsc/tree/main/csv-compressor)
-Those files would work as input for the compressor.
-
-Compressor usage:
-
-```txt
-Usage: atsc [OPTIONS] <INPUT>
-
-Arguments:
-  <INPUT>  input file
-
-      --compressor <COMPRESSOR>
-          Select a compressor, default is auto [default: auto] [possible values: auto, noop, fft, constant, polynomial, idw]
-  -e, --error <ERROR>
-          Sets the maximum allowed error for the compressed data, must be between 0 and 50. Default is 5 (5%).
-          0 is lossless compression
-          50 will do a median filter on the data.
-          In between will pick optimize for the error [default: 5]
-  -u
-          Uncompresses the input file/directory
-  -c, --compression-selection-sample-level <COMPRESSION_SELECTION_SAMPLE_LEVEL>
-          Samples the input data instead of using all the data for selecting the optimal compressor.
-          Only impacts speed, might or not increased compression ratio. For best results use 0 (default).
-          Only works when compression = Auto.
-          0 will use all the data (slowest)
-          6 will sample 128 data points (fastest) [default: 0]
-      --verbose
-          Verbose output, dumps everysample in the input file (for compression) and in the ouput file (for decompression)
-      --csv
-          Defines user input as a CSV file
-      --no-header
-          Defines if the CSV has no header
-      --fields <FIELDS>
-          Defines names of fields in CSV file. It should follow this format:
-            --fields=TIME_FIELD_NAME,VALUE_FIELD_NAME
-          It assumes that the one before comma is a name of time field and the one
-          after comma is value field. [default: time,value]
-  -h, --help
-          Print help
-  -V, --version
-          Print version
+ATSC accepts WBRO input, or CSV input with `--csv`, and writes BRO v1 without
+changing the wire format. Prefer the explicit commands:
+
+```text
+atsc inspect <INPUT> [--json]
+atsc verify <INPUT>
+atsc compress <INPUT> [-o <OUTPUT>] [OPTIONS]
+atsc decompress <INPUT> [-o <OUTPUT>]
 ```
 
-#### Compress a File
+Explicit-command options belong after their subcommand. Combining an explicit
+subcommand with legacy root input/options is a usage error.
 
-To compress a file using ATSC, run:
+Examples:
 
 ```bash
-atsc <input-file>
+atsc inspect metrics.bro
+atsc inspect metrics.bro --json
+atsc verify metrics.bro
+atsc compress metrics.wbro --compressor rle
+atsc compress metrics.csv --csv --fields=time,value -o metrics.bro
+atsc decompress metrics.bro -o restored.wbro
 ```
 
-#### Decompress a File
+Compression defaults to automatic codec selection with a 3% maximum error.
+Directory input is supported for compression and decompression; ATSC snapshots
+and filters the initial entries, rejects duplicate derived output paths, then
+processes each eligible input once.
+`-o/--output` is only valid for a single input file.
+BRO, WBRO, and CSV reads default to 256 MiB and 33,423,360 samples. WBRO
+metadata and CSV structure are validated before unbounded allocations, and
+non-finite encoder inputs report their sample index. File open/read/write
+failures retain the I/O exit status.
 
-To decompress a file, use:
+Existing invocations remain available as compatibility mode and use the same
+safe implementation:
 
 ```bash
-atsc -u <input-file>
+atsc [OPTIONS] <INPUT>
+atsc -u <INPUT>
+```
+
+For a legacy file named `inspect`, `verify`, `compress`, or `decompress`, use
+`atsc -- <name>` or an explicit path such as `atsc ./inspect`. ATSC does not
+guess command intent from filesystem existence.
+
+Library callers can use bounded parsing and a reusable decoder directly:
+
+```rust
+use atsc::{data::CompressedStream, decoder::Decoder};
+
+let bytes = std::fs::read("metrics.bro")?;
+let stream = CompressedStream::try_from_bytes(&bytes)?;
+let mut decoder = Decoder::new();
+let values = decoder.decode_range(&stream, 1_000..2_000)?;
 ```
 
+BRO v1's serialized `frame_size` value is opaque legacy host-layout metadata,
+not an encoded frame length, and is intentionally ignored while decoding.
+An authoritative frame byte length requires a future BRO v2 field.
+
+See [the usage guide](docs/usage.md) for all compression options, output naming,
+JSON behavior, and stable exit codes.
+
 ## Releases
 
 ### v0.7 - 20/11/2024

atsc/Cargo.toml

@@ -3,6 +3,7 @@ name = "atsc"
 version = "0.7.2"
 authors = ["Carlos Rolo <carlos.rolo@netapp.com>"]
 edition = "2021"
+rust-version = "1.81"
 license = "Apache-2.0"
 description = "An Advanced Time-Series Compressor"
 
@@ -24,6 +25,8 @@ inverse_distance_weight = "0.1.1"
 num-traits = "0.2"
 csv = "1.3.1"
 thiserror = "2.0.3"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
 
 [dev-dependencies]
 criterion = "0.5.1"
@@ -36,3 +39,7 @@ harness= false
 [[bench]]
 name = "polynomial_bench"
 harness= false
+
+[[bench]]
+name = "decompression_bench"
+harness = false