diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac293bf..4de4992 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,26 +28,19 @@ jobs: # hashdb leg) discards the already-built all-features artifacts and the next # run pays the full cold build again. Persist on failure to break that trap. cache-on-failure: true - # etcd-client (opendal services-etcd) requires protoc at build time. # Linux linker: neither mold nor the default GNU ld can link the ~100 huge # bundled-C++ test binaries on the 7 GB runner — mold DEADLOCKS (6 h hang) and # GNU ld SIGBUSes (collect2: ld terminated with signal 7, core dumped, proven in # CI logs). Use lld, which links these large binaries memory-efficiently without # mold's deadlock; the debuginfo reduction below removes the actual memory driver. - - name: Install protoc + lld (Linux) + - name: Install lld (Linux) if: runner.os == 'Linux' - run: sudo apt-get update -qq && sudo apt-get install -y protobuf-compiler clang lld + run: sudo apt-get update -qq && sudo apt-get install -y clang lld - name: Use lld linker (Linux) if: runner.os == 'Linux' run: | mkdir -p ~/.cargo printf '[target.x86_64-unknown-linux-gnu]\nlinker = "clang"\nrustflags = ["-C", "link-arg=-fuse-ld=lld"]\n' >> ~/.cargo/config.toml - - name: Install protoc (macOS) - if: runner.os == 'macOS' - run: brew install protobuf - - name: Install protoc (Windows) - if: runner.os == 'Windows' - run: choco install protoc --no-progress # blazehash links ~800 crates + bundled C++ (duckdb/rocksdb/wasmtime/yara-x) into # each of ~100 integration-test binaries. With full debuginfo (debuginfo=2) the # linker exhausts the runner's RAM: link.exe LNK1102 on Windows, and collect2/ld @@ -81,8 +74,6 @@ jobs: with: components: llvm-tools-preview - uses: taiki-e/install-action@a402910a723481c4c80d006d75298c796a9c8695 # cargo-llvm-cov - - name: Install protoc - run: sudo apt-get update -qq && sudo apt-get install -y protobuf-compiler - name: Generate coverage run: cargo llvm-cov --all-features --lcov --output-path lcov.info - name: Upload to Codecov @@ -98,8 +89,6 @@ jobs: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: dtolnay/rust-toolchain@1.88.0 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - - name: Install protoc - run: sudo apt-get update -qq && sudo apt-get install -y protobuf-compiler - run: cargo check --all-features deny: diff --git a/Cargo.toml b/Cargo.toml index f0bf126..1d86507 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,6 +146,10 @@ signature = "2" qrcode = { version = "0.14", optional = true } image = { version = "0.25", optional = true, default-features = false, features = ["png"] } regex = "1" +# Scoped to forensically-relevant evidence-transfer targets — the stores DFIR +# collectors (Velociraptor, KAPE) read case evidence from and write collections +# back to. Native DB/KV engines, caches, dev/ML/CI-cache, decentralized and +# consumer-drive backends are intentionally excluded (see ADR-0010). opendal = { version = "0.55.0", optional = true, features = [ # Cloud object storage "services-s3", # AWS S3, MinIO, R2, Wasabi, Backblaze B2 (S3-compat) @@ -158,66 +162,24 @@ opendal = { version = "0.55.0", optional = true, features = [ "services-obs", # Huawei Cloud OBS "services-oss", # Alibaba Cloud OSS "services-swift", # OpenStack Swift - "services-upyun", # Upyun CDN storage - # Consumer / enterprise cloud drives + # Cloud drive (the one drive with dedicated wiring in src/remote/gdrive/) "services-gdrive", # Google Drive - "services-onedrive", # Microsoft OneDrive - "services-dropbox", # Dropbox - "services-aliyun-drive",# Aliyun Drive (Alibaba) - "services-yandex-disk", # Yandex Disk - "services-pcloud", # pCloud - "services-koofr", # Koofr - "services-seafile", # Seafile - # Developer / ML / infra - "services-github", # GitHub (code forensics) - "services-huggingface", # HuggingFace datasets/models - "services-vercel-artifacts", # Vercel build cache - "services-vercel-blob", # Vercel Blob - # Distributed / big data - "services-alluxio", # Alluxio data orchestration + # Hadoop "services-webhdfs", # WebHDFS REST (no JVM needed) - "services-lakefs", # LakeFS data versioning - "services-dbfs", # Databricks DBFS - "services-ghac", # GitHub Actions Cache (useful in CI forensics) - # Decentralized - "services-ipfs", # IPFS (content-addressed, prost dep is pure Rust) - "services-ipmfs", # IPFS Mutable File System - # In-memory / embedded (useful for testing + pipeline) - "services-memory", - "services-dashmap", # Concurrent in-memory map - "services-mini-moka", # Bounded in-memory cache - "services-moka", # Full-featured in-memory cache with TTL - "services-redb", # Embedded key-value (pure Rust, ACID) - "services-sled", # Embedded key-value (pure Rust) - "services-cacache", # Content-addressable disk cache - # Network key-value / databases - "services-redis", # Redis - "services-redis-native-tls", # Redis with TLS (rediss://) - "services-memcached", # Memcached - "services-etcd", # etcd (gRPC, pure Rust) - "services-tikv", # TiKV distributed KV - "services-mongodb", # MongoDB (pure Rust driver) - "services-gridfs", # MongoDB GridFS - "services-mysql", # MySQL / MariaDB (sqlx, pure Rust) - "services-postgresql", # PostgreSQL (sqlx, pure Rust) - "services-sqlite", # SQLite (sqlx, pure Rust) - # "services-surrealdb" — excluded: pulls async-graphql@7.2.1 which requires rustc 1.89 - "services-persy", # Persy embedded (pure Rust) - # Misc - "services-cloudflare-kv", # Cloudflare KV (via REST) - "services-d1", # Cloudflare D1 SQLite (via REST) - # Filesystem + "services-hdfs-native", # HDFS pure-Rust native client (no Java/libhdfs required) + # SQL (sqlx, pure Rust) + "services-mysql", # MySQL / MariaDB + "services-postgresql", # PostgreSQL (README-advertised) + "services-sqlite", # SQLite + # Filesystem / protocol "services-fs", "services-http", "services-webdav", - # services-sftp intentionally omitted: openssh Rust crate is Unix-only. + "services-memory", # in-memory (useful for testing + pipeline) + # services-sftp intentionally omitted: openssh Rust crate is Unix-only. # SFTP is handled via ssh2 (libssh2 vendored, cross-platform) in src/remote/sftp.rs. # services-ftp intentionally omitted: async-tls/tokio-rustls conflict. # FTP/FTPS is handled via suppaftp (blocking + rustls) instead. - "services-hdfs-native", # HDFS pure-Rust native client (no Java/libhdfs required) - "services-compfs", # compio-based async filesystem - # services-monoiofs intentionally omitted from main features: monoio requires Linux io_uring. - # Added via [target.'cfg(target_os = "linux")'.dependencies] below. "blocking", ] } @@ -231,11 +193,6 @@ libc = "0.2" [target.'cfg(target_os = "linux")'.dependencies] libc = "0.2" -# monoiofs depends on monoio (io_uring) which only compiles on Linux. -# Optional + gated behind the `remote` feature (same as the main opendal dep): -# the default build must not pull the opendal cloud stack (and its vulnerable -# DNS / quick-xml transitive tree) on Linux either. -opendal = { version = "0.55.0", features = ["services-monoiofs"], optional = true } [target.'cfg(target_os = "windows")'.dependencies] windows-sys = { version = "0.59", features = [ @@ -272,7 +229,6 @@ ots = [] pq = [] qr = ["dep:qrcode", "dep:image"] remote = ["dep:opendal", "dep:tokio", "dep:suppaftp", "dep:ssh2"] -rocksdb-storage = ["remote", "opendal/services-rocksdb"] [package.metadata.deb] maintainer = "SecurityRonin " diff --git a/README.md b/README.md index 116eec9..eb757d9 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Every hashdeep flag works. Your existing scripts keep working. These are the add | Duplicate detection | Y | — | | Manifest diff / merge / update | Y | — | | Live monitoring (watch) | Y | — | -| Remote storage (S3/GCS/Azure/WebDAV/60+ protocols) | Y | — | +| Remote storage (S3/GCS/Azure/SFTP/WebDAV/HDFS/SQL) | Y | — | | Google Drive hash-without-download | Y | — | | GPU-accelerated SHA-256/MD5 | Y | — | | MCP server (AI-assisted workflows) | Y | — | @@ -137,7 +137,7 @@ BLAKE3 runs at **1,640–1,780 MB/s** — 2.8x faster than hashdeep's best algor ## Remote Storage -Evidence doesn't live only on disk. blazehash speaks 50+ storage protocols natively — the same command works whether the data is local, on S3, in Google Drive, or on an SFTP server. +Evidence doesn't live only on disk. blazehash reads and writes remote storage natively — the same command works whether the data is local, on S3, in Google Drive, or on an SFTP server. ```bash # Hash an S3 prefix directly @@ -160,7 +160,7 @@ blazehash sftp://admin@192.168.1.10/cases/image.dd Credentials come from standard environment variables — `AWS_ACCESS_KEY_ID`, `GOOGLE_APPLICATION_CREDENTIALS`, `AZURE_STORAGE_ACCOUNT` — so existing tooling and CI secrets work without changes. For Google Drive, run `blazehash gdrive auth login` once. -Supported: S3, GCS, Azure Blob/Files/ADLS, Backblaze B2, Tencent COS, Huawei OBS, Alibaba OSS, OpenStack Swift, OneDrive, Dropbox, Google Drive, pCloud, Yandex Disk, SFTP, FTP, WebDAV, WebHDFS, Redis, MongoDB, PostgreSQL, and [30+ more →](https://securityronin.github.io/blazehash/remote/) +Supported: S3, GCS, Azure Blob/Files/ADLS, Backblaze B2, Tencent COS, Huawei OBS, Alibaba OSS, OpenStack Swift, Google Drive, HDFS/WebHDFS, MySQL, PostgreSQL, SQLite, SFTP, FTP/FTPS, WebDAV, and HTTP — [full reference →](https://securityronin.github.io/blazehash/remote/) --- @@ -174,7 +174,7 @@ cargo install blazehash --all-features | Flag | Default | Enables | |------|:-------:|---------| -| `remote` | on | Remote storage (S3/GCS/Azure/SFTP/GDrive/60+ protocols) | +| `remote` | on | Remote storage (S3/GCS/Azure/SFTP/GDrive, HDFS, SQL) | | `sqlite` | on | `--format sqlite` output + NSRL SQLite database | | `parquet-output` | on | `--format parquet` output | | `duckdb-output` | on | `--format duckdb` output | diff --git a/deny.toml b/deny.toml index 5c42d53..4b32e1c 100644 --- a/deny.toml +++ b/deny.toml @@ -3,11 +3,10 @@ targets = [] # The `remote` feature (opendal cloud/object-storage stack) is a genuinely-optional # heavy subsystem — opt-in, NOT default (see Cargo.toml [features]). It drags in a # vulnerable transitive tree (hickory DNS RUSTSEC-2026-0118/0119, quick-xml -# RUSTSEC-2026-0194/0195, plus sqlx/tikv/rustls chains) that no hashing workload +# RUSTSEC-2026-0194/0195, plus sqlx/rustls chains) that no hashing workload # exercises. Rather than blanket-ignore all of those advisories, we keep opendal -# out of the audited graph entirely: check every OTHER feature, never `remote` -# (or `rocksdb-storage`, which pulls it in). `cargo deny check` therefore audits -# exactly what a default `cargo build` ships. +# out of the audited graph entirely: check every OTHER feature, never `remote`. +# `cargo deny check` therefore audits exactly what a default `cargo build` ships. all-features = false features = [ "forensic-image", diff --git a/docs/PRD.md b/docs/PRD.md index 5c00fd3..332a18f 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -80,7 +80,7 @@ Grounded in `src/` and `src/cli.rs`: (`duckdb-output`, bundled engine floored for the MSVC toolchain — [ADR 0011](decisions/0011-duckdb-msvc-fmt-floor.md)), JSON/JSONL, STIX 2.1, and ECS NDJSON (`src/format/`, `src/output.rs`). -- **Remote storage:** S3/GCS/Azure/60+ backends via opendal, plus FTP/SFTP, and +- **Remote storage:** S3/GCS/Azure and other object stores via opendal, plus HDFS, SQL, FTP/SFTP, and Google Drive hash-without-download (`src/remote/`, `remote` feature — [ADR 0002](decisions/0002-batteries-included-remote-opt-in.md)). - **Interfaces:** the `blazehash` CLI, an interactive `tui` dashboard diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 9ea3776..5267952 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -951,7 +951,7 @@ blazehash completions man > blazehash.1 && man ./blazehash.1 ## Remote storage -blazehash accepts remote URIs for both input paths and `-o` output — 50+ protocols supported. +blazehash accepts remote URIs for both input paths and `-o` output — object stores, HDFS, SQL, SFTP/FTP, and WebDAV/HTTP. **Full protocol reference:** [Remote Storage →](remote.md) @@ -974,12 +974,9 @@ blazehash gdrive://1ABCdef... blazehash sftp://admin@192.168.1.10/evidence/disk.dd ``` -Supported schemes include: `s3`, `gcs`, `azblob`, `azdls`, `azfile`, `b2`, `cos`, `obs`, `oss`, -`swift`, `upyun`, `gdrive`, `onedrive`, `dropbox`, `aliyun-drive`, `yandex-disk`, `pcloud`, -`koofr`, `seafile`, `github`, `huggingface`, `vercel-blob`, `alluxio`, `hdfs`, `webhdfs`, `lakefs`, -`ipfs`, `ipmfs`, `redis`, `rediss`, `memcached`, `etcd`, `tikv`, `mongodb`, `mysql`, `postgresql`, -`sqlite`, `rocksdb` (opt-in), `cloudflare-kv`, `d1`, `sftp`, `ftp`, `ftps`, `webdav`, `http`, -`https`, `compfs`, `monoiofs` (Linux), `file`, `mem`. +Supported schemes: `s3`, `gcs`, `azblob`, `azdls`, `azfile`, `b2`, `cos`, `obs`, `oss`, +`swift`, `gdrive`, `hdfs`, `webhdfs`, `mysql`, `postgresql`, `sqlite`, `sftp`, `ftp`, `ftps`, +`webdav`, `http`, `https`, `file`, `mem`. --- diff --git a/docs/decisions/0012-scope-remote-opendal-backends.md b/docs/decisions/0012-scope-remote-opendal-backends.md new file mode 100644 index 0000000..28da678 --- /dev/null +++ b/docs/decisions/0012-scope-remote-opendal-backends.md @@ -0,0 +1,91 @@ +# 12. Scope the `remote` OpenDAL backend set to forensically-relevant evidence-transfer targets + +Date: 2026-07-27 +Status: Accepted + +## Context + +ADR-0002 gated the whole `opendal` cloud stack behind the opt-in `remote` +feature, so the default binary and the security audit stay opendal-free. It did +not scope *which* OpenDAL services `remote` compiles — every service OpenDAL +publishes (~50) was enabled. That set carries native database/KV engines and an +exotic long tail that no evidence-transfer workflow exercises but that costs real +build time: + +- `services-rocksdb` pulls `librocksdb-sys`, a bundled C++ build — the + `link.exe LNK1102` OOM culprit on the Windows leg under a debuginfo build. +- `services-etcd` / `services-tikv` pull `tonic`/`prost` gRPC stacks that need a + `protoc` binary at build time — the sole reason `ci.yml` installs + `protobuf-compiler` on all three OSes. +- `services-mongodb`/`gridfs` and the cache / embedded-KV set (`redis`, + `memcached`, `sled`, `redb`, `persy`, `moka`, `mini-moka`, `dashmap`, + `cacache`) add breadth with no forensic evidence-transfer story. + +blazehash's `remote` feature exists so an examiner can hash evidence that lives +on a remote store, and write a collection back to one — the same targets DFIR +collectors (Velociraptor, KAPE) upload to: object storage (S3/Azure/GCS), +SFTP/FTP, WebHDFS/HDFS, and SQL stores. A distributed-KV cache or a +decentralized-filesystem gateway is not where case evidence lives. + +Two honesty notes on the analysis behind this decision. An initial review pass +claimed specific dependency-crate-count reductions and a `libsqlite3-sys` +removal; a Codex critic could not reproduce those counts (`--all-features` +measured ~771 crates, not the cited figure) and `libsqlite3-sys` is pulled by +`rusqlite` regardless — so no crate-count figure is asserted here. The +load-bearing, verifiable claim is narrower and structural: dropping +`rocksdb`/`etcd`/`tikv` removes the `librocksdb-sys` C++ build and the +`tonic`/`prost` chains, which lets the `protoc` install step leave CI entirely. +The compile-time and linker-RSS win is confirmed by the before/after CI +wall-clock on the enactment PR, not by a crate count. + +## Decision + +Scope the `remote` feature's OpenDAL service set to forensically-relevant +evidence-transfer targets. `remote` stays opt-in and remote-free-by-default +(ADR-0002 unchanged); this ADR narrows *which* services it compiles. + +**KEEP** — evidence-transfer targets: + +- Object storage: `s3`, `gcs`, `azblob`, `azdls`, `azfile`, `b2`, `cos`, `obs`, + `oss`, `swift` +- Cloud drive: `gdrive` — the one drive with dedicated wiring + (`src/remote/gdrive/`) +- Hadoop: `webhdfs`, `hdfs-native` (pure-Rust, no JVM) +- SQL: `mysql`, `postgresql`, `sqlite` (sqlx; PostgreSQL is README-advertised) +- Local / protocol: `fs`, `memory`, `http`, `webdav`, plus SFTP (`ssh2`) and + FTP (`suppaftp`), which are non-opendal by design (ADR-0002 §4) + +**DROP**: + +- Native DB/KV + caches: `rocksdb` (and the `rocksdb-storage` feature), `etcd`, + `tikv`, `mongodb`, `gridfs`, `redis` (+`redis-native-tls`), `memcached`, + `sled`, `redb`, `persy`, `moka`, `mini-moka`, `dashmap`, `cacache` +- Dev / ML / CI-cache: `github`, `huggingface`, `vercel-blob`, + `vercel-artifacts`, `ghac` +- Big data beyond WebHDFS/HDFS: `alluxio`, `lakefs`, `dbfs` +- Decentralized: `ipfs`, `ipmfs` +- Alternate-runtime filesystems: `compfs`, `monoiofs` +- Cloudflare: `cloudflare-kv`, `d1` +- Regional / consumer drives: `onedrive`, `dropbox`, `aliyun-drive`, + `yandex-disk`, `pcloud`, `koofr`, `seafile`, `upyun` + +On the drives specifically: GDrive is kept because it carries dedicated wiring; +OneDrive and Dropbox had only generic scheme wiring (no CLI or test path), so +absent a concrete case they ship as unexercised capability and are dropped — +one line to re-add if a real need appears. + +## Consequences + +- The `protoc` build-tool install step is removed from `ci.yml` (no + `etcd`/`tikv` gRPC stack remains to need it). +- `librocksdb-sys` leaves the `--features remote` graph, removing the + Windows-linker memory pressure it contributed. +- `rocksdb-storage` is removed from `[features]`, and `deny.toml`'s per-feature + enumeration drops it. +- `operator_for_uri` loses the ~25 dropped scheme arms; the KEEP schemes and the + informative SFTP/FTP bail arms remain. +- The cost win — smaller `remote` compile, lower Windows-linker RSS — is proven + by the enactment PR's CI wall-clock against the prior run, not asserted as a + crate-count figure. +- Re-adding any dropped backend is a one-line feature plus one match arm; nothing + here is one-way. diff --git a/docs/index.md b/docs/index.md index bc4105e..e5481de 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ The only open-source forensic hashing tool that answers every question a court asks about digital evidence — *what* changed, *who* handled it, *when* it was sealed, and *in what context* — in a single binary that's drop-in compatible with hashdeep. -Now with **50+ remote storage backends** built in via Apache OpenDAL: hash from S3, GCS, Azure Blob, WebDAV, SFTP, and more — no extra flags required. Hash Google Drive files in memory via `gdrive://` URI — no local copy needed. +Read and write evidence directly from **remote storage** via Apache OpenDAL — S3, GCS, Azure Blob, WebDAV, SFTP, HDFS, and SQL stores, the targets DFIR collectors actually use — built into the release binaries, no flag at runtime. Hash Google Drive files in memory via a `gdrive://` URI, no local copy needed. ```bash # Acquire evidence with chain-of-custody metadata diff --git a/docs/remote.md b/docs/remote.md index 0183f74..e324981 100644 --- a/docs/remote.md +++ b/docs/remote.md @@ -1,17 +1,27 @@ # Remote Storage -blazehash speaks **50+ storage protocols** natively via [Apache OpenDAL](https://opendal.apache.org/). Any URI that resolves to file-like data is a valid input path or `-o` output target — no plugins, no adapters, no staging. +blazehash reads and writes evidence directly from remote storage via +[Apache OpenDAL](https://opendal.apache.org/), scoped to the backends where +forensic evidence actually lives and where DFIR collectors (Velociraptor, KAPE) +deposit it: object storage, SFTP/FTP, Hadoop, SQL stores, and WebDAV/HTTP. Any +URI that resolves to file-like data is a valid input path or `-o` output target — +no plugins, no adapters, no staging. ```bash # Read from remote, write manifest to remote — entirely off-disk blazehash s3://dfir-bucket/case-001/ -o gcs://evidence-archive/case-001.hash ``` +The remote stack is an opt-in build feature (`--features remote`); the release +binaries enable it. ADR-0002 records why it is opt-in and ADR-0012 why the +backend set is scoped to evidence-transfer targets (both in the project's +`docs/decisions/`). + --- ## Cloud Object Storage -These are the most common backends for evidence archiving. +The most common backends for evidence archiving. | Scheme | Backend | Auth env vars | |--------|---------|---------------| @@ -25,7 +35,6 @@ These are the most common backends for evidence archiving. | `obs://bucket/key` | Huawei Cloud OBS | `HUAWEI_ACCESS_KEY_ID`, `HUAWEI_SECRET_ACCESS_KEY`, `HUAWEI_REGION` | | `oss://bucket/key` | Alibaba Cloud OSS | `ALIBABA_CLOUD_ACCESS_KEY_ID`, `ALIBABA_CLOUD_ACCESS_KEY_SECRET`, `ALIBABA_CLOUD_REGION` | | `swift://container/path` | OpenStack Swift | `SWIFT_ENDPOINT`, `SWIFT_TOKEN` | -| `upyun://bucket/key` | Upyun CDN storage | `UPYUN_OPERATOR`, `UPYUN_PASSWORD` | ### S3-compatible endpoints @@ -47,24 +56,13 @@ blazehash s3://my-bucket/evidence/ --- -## Cloud Drives +## Google Drive -Consumer and enterprise cloud drives — useful when evidence is a file shared from a suspect's or custodian's account. +Useful when evidence is a file shared from a suspect's or custodian's account. blazehash uses the Drive API to hash without staging the file locally: | Scheme | Backend | Auth | |--------|---------|------| -| `gdrive://file-id` | Google Drive | Run `blazehash gdrive auth login` once to cache OAuth2 token | -| `onedrive://path` | Microsoft OneDrive | `ONEDRIVE_ACCESS_TOKEN` | -| `dropbox://path` | Dropbox | `DROPBOX_ACCESS_TOKEN` | -| `aliyun-drive://path` | Aliyun Drive (Alibaba) | `ALIYUN_DRIVE_ACCESS_TOKEN` | -| `yandex-disk://path` | Yandex Disk | `YANDEX_DISK_ACCESS_TOKEN` | -| `pcloud://path` | pCloud | `PCLOUD_USERNAME`, `PCLOUD_PASSWORD`, `PCLOUD_ENDPOINT` | -| `koofr://path` | Koofr | `KOOFR_EMAIL`, `KOOFR_PASSWORD`, `KOOFR_ENDPOINT` | -| `seafile://server/repo/path` | Seafile | `SEAFILE_USERNAME`, `SEAFILE_PASSWORD` | - -### Google Drive without downloading - -blazehash uses the Drive API to hash without staging the file locally: +| `gdrive://file-id` | Google Drive | Run `blazehash gdrive auth login` once to cache the OAuth2 token | ```bash # By file ID @@ -77,39 +75,16 @@ blazehash https://drive.google.com/file/d/1Ykbd9fDXxWnD1-MTag_-8-Wh_Wnd28q0/view blazehash gdrive auth login ``` -### OneDrive / SharePoint - -```bash -export ONEDRIVE_ACCESS_TOKEN=$(az account get-access-token --resource https://graph.microsoft.com -o tsv --query accessToken) -blazehash onedrive://Documents/CaseFiles/image.dd -``` - --- -## Developer / ML / Infra +## Hadoop / HDFS -| Scheme | Backend | Auth | -|--------|---------|------| -| `github://owner/repo/path` | GitHub (raw file access) | `GITHUB_TOKEN` | -| `huggingface://owner/repo/path` | HuggingFace datasets and models | `HUGGINGFACE_TOKEN` | -| `vercel-blob://key` | Vercel Blob storage | `BLOB_READ_WRITE_TOKEN` | -| `vercel-artifacts://key` | Vercel build artifact cache | `VERCEL_ARTIFACTS_TOKEN` | -| `ghac://key` | GitHub Actions Cache | Set in GitHub Actions environment | -| `dbfs://path` | Databricks DBFS | `DATABRICKS_HOST`, `DATABRICKS_TOKEN` | -| `alluxio://host:port/path` | Alluxio data orchestration | — | -| `webhdfs://host:port/path` | WebHDFS REST (Hadoop — no JVM required) | `WEBHDFS_USER` | -| `lakefs://repo/branch/path` | LakeFS data versioning | `LAKEFS_ACCESS_KEY_ID`, `LAKEFS_SECRET_ACCESS_KEY`, `LAKEFS_ENDPOINT` | -| `ipfs://CID/path` | IPFS content-addressed storage | `IPFS_GATEWAY` (default: `http://127.0.0.1:8080`) | -| `ipmfs:///path` | IPFS Mutable File System | `IPFS_ENDPOINT` (default: `http://127.0.0.1:5001`) | - -### Hadoop / HDFS - -Two schemes are available depending on your cluster setup: +Two schemes, depending on your cluster setup — both pure-Rust, no JVM and no `libhdfs`: | Scheme | Backend | Notes | |--------|---------|-------| -| `hdfs://namenode:port/path` | HDFS (pure-Rust native client) | No JVM, no `libhdfs` — uses `hdfs-native` crate speaking Hadoop RPC | -| `webhdfs://namenode:port/path` | WebHDFS REST API | No JVM, works with any Hadoop 2.x+ namenode | +| `hdfs://namenode:port/path` | HDFS (pure-Rust native client) | Speaks Hadoop RPC via the `hdfs-native` crate | +| `webhdfs://host:port/path` | WebHDFS REST API | Works with any Hadoop 2.x+ namenode; `WEBHDFS_USER` for the user name | ```bash # Pure-Rust HDFS native client (no Java required) @@ -121,41 +96,21 @@ blazehash webhdfs://namenode.corp:50070/user/evidence/case-001/ `hdfs://` is preferred when the cluster exposes the native Hadoop RPC port (default 8020/9000). `webhdfs://` is the fallback when only the HTTP REST endpoint is reachable. -### GitHub (code forensics) - -Hash the exact state of a repository path at HEAD (or any ref via the API): - -```bash -export GITHUB_TOKEN=ghp_... -blazehash github://octocat/Hello-World/README -``` - --- -## Network KV / Databases +## SQL Databases Useful when evidence artifacts are stored in operational datastores rather than file systems. | Scheme | Backend | Auth / connection | |--------|---------|-------------------| -| `redis://host/key` | Redis | Standard Redis URL (supports `redis://[:password@]host:port`) | -| `memcached://host/key` | Memcached | `tcp://host:port` | -| `etcd://host/key` | etcd (gRPC) | `ETCD_USERNAME`, `ETCD_PASSWORD` | -| `tikv://pd-host/key` | TiKV distributed KV | PD endpoint | -| `mongodb://host/db/collection/key` | MongoDB | Standard MongoDB connection string | -| `gridfs://host/db/bucket/key` | MongoDB GridFS | Standard MongoDB connection string | | `mysql://host/db/key` | MySQL / MariaDB | Standard DSN | | `postgresql://host/db/key` | PostgreSQL | Standard DSN | | `sqlite://path/to.db/key` | SQLite (file on disk) | File path | -| `cloudflare-kv://namespace/key` | Cloudflare Workers KV | `CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_API_TOKEN` | -| `d1://database-id/key` | Cloudflare D1 (SQLite via REST) | `CLOUDFLARE_ACCOUNT_ID`, `CLOUDFLARE_API_TOKEN` | ```bash -# Hash a value stored in Redis -blazehash redis://localhost/evidence:case-001:image - -# Hash a MongoDB document field -blazehash mongodb://localhost/forensics/artifacts/abc123 +# Hash a value stored in a PostgreSQL row +blazehash postgresql://user:pass@localhost/forensics/artifacts ``` --- @@ -194,51 +149,7 @@ blazehash sftp://admin@192.168.1.10/evidence/disk.dd | Scheme | Backend | Notes | |--------|---------|-------| | `mem://bucket/key` | In-process memory | Ephemeral; useful in tests and pipeline stages | -| `redis://...` | Redis (plaintext) | Also serves as fast shared cache between pipeline stages | -| `rediss://host:port/key` | Redis with TLS | Same as `redis://` but over an encrypted connection | -| `sqlite://path/db/key` | SQLite file | Lightweight embedded KV; good for offline pipelines | -| `rocksdb:///path/to/db/key` | RocksDB embedded KV | Requires `--features rocksdb-storage` (compile-time opt-in) | - -### Redis TLS (`rediss://`) - -The `rediss://` scheme (double-s) connects to Redis over TLS — useful when your Redis instance requires encrypted connections (e.g. Redis Cloud, Upstash, or self-hosted with TLS). - -```bash -# Hash a value stored in Redis over TLS -blazehash rediss://redis.cloud.example.com:6380/evidence:case-001:hash -``` - -### RocksDB (`rocksdb://`) - -RocksDB support is an optional compile-time feature — it pulls in the RocksDB C++ library at build time. Not included in the default binary; build with: - -```bash -cargo install blazehash --features rocksdb-storage -``` - -```bash -# Hash a value stored in a local RocksDB database -blazehash rocksdb:///var/lib/evidence/casedb/artifact-001 -``` - -## Compio / Monoio Async Filesystems - -These backends replace the standard OS filesystem (`file://`) with alternative async I/O runtimes for workloads where kernel-level async matters. - -| Scheme | Backend | Platform | -|--------|---------|----------| -| `compfs:///abs/path/file` | compio filesystem | macOS, Linux, Windows (io_uring / kqueue / IOCP) | -| `monoiofs:///abs/path/file` | monoio filesystem | Linux only (io_uring) | - -```bash -# Hash via compio (cross-platform async I/O) -blazehash compfs:///mnt/evidence/disk.dd - -# Hash via monoio (Linux io_uring — lower overhead for large sequential reads) -blazehash monoiofs:///mnt/evidence/disk.dd -``` - -In practice, for most forensic workloads the default filesystem is sufficient. These backends are relevant when integrating blazehash into a compio or monoio async pipeline. +| `sqlite://path/db/key` | SQLite file | Lightweight embedded store; good for offline pipelines | --- @@ -283,31 +194,6 @@ blazehash -a -k s3://dfir-bucket/case-001.hash -r /mnt/evidence | `ALIBABA_CLOUD_REGION` | oss | | `SWIFT_ENDPOINT` | swift | | `SWIFT_TOKEN` | swift | -| `UPYUN_OPERATOR` | upyun | -| `UPYUN_PASSWORD` | upyun | -| `ONEDRIVE_ACCESS_TOKEN` | onedrive | -| `DROPBOX_ACCESS_TOKEN` | dropbox | -| `ALIYUN_DRIVE_ACCESS_TOKEN` | aliyun-drive | -| `YANDEX_DISK_ACCESS_TOKEN` | yandex-disk | -| `PCLOUD_USERNAME` / `PCLOUD_PASSWORD` | pcloud | -| `PCLOUD_ENDPOINT` | pcloud (default: `https://api.pcloud.com`) | -| `KOOFR_EMAIL` / `KOOFR_PASSWORD` | koofr | -| `KOOFR_ENDPOINT` | koofr (default: `https://app.koofr.net`) | -| `SEAFILE_USERNAME` / `SEAFILE_PASSWORD` | seafile | -| `SEAFILE_REPO` | seafile (repo name, default: `My Library`) | -| `GITHUB_TOKEN` | github | -| `HUGGINGFACE_TOKEN` | huggingface | -| `BLOB_READ_WRITE_TOKEN` | vercel-blob | -| `VERCEL_ARTIFACTS_TOKEN` | vercel-artifacts | -| `DATABRICKS_HOST` | dbfs | -| `DATABRICKS_TOKEN` | dbfs | | `WEBHDFS_USER` | webhdfs | -| `LAKEFS_ACCESS_KEY_ID` | lakefs | -| `LAKEFS_SECRET_ACCESS_KEY` | lakefs | -| `LAKEFS_ENDPOINT` | lakefs (default: `http://localhost:8000`) | -| `IPFS_GATEWAY` | ipfs (default: `http://127.0.0.1:8080`) | -| `IPFS_ENDPOINT` | ipmfs (default: `http://127.0.0.1:5001`) | -| `CLOUDFLARE_ACCOUNT_ID` | cloudflare-kv, d1 | -| `CLOUDFLARE_API_TOKEN` | cloudflare-kv, d1 | | `BLAZEHASH_SFTP_KEY_PATH` | sftp | | `BLAZEHASH_SFTP_KNOWN_HOSTS_STRATEGY` | sftp (`add` \| `strict` \| `accept_new`) | diff --git a/src/remote/mod.rs b/src/remote/mod.rs index 89632dd..e27316d 100644 --- a/src/remote/mod.rs +++ b/src/remote/mod.rs @@ -96,8 +96,6 @@ pub enum UriScheme { PostgreSQL, SQLite, SurrealDB, - #[cfg(feature = "rocksdb-storage")] - RocksDb, // Misc CloudflareKv, D1, @@ -163,8 +161,6 @@ impl UriScheme { "postgresql" => Some(Self::PostgreSQL), "sqlite" => Some(Self::SQLite), "surrealdb" => Some(Self::SurrealDB), - #[cfg(feature = "rocksdb-storage")] - "rocksdb" => Some(Self::RocksDb), "cloudflare-kv" => Some(Self::CloudflareKv), "d1" => Some(Self::D1), "http" => Some(Self::Http), diff --git a/src/remote/operator.rs b/src/remote/operator.rs index 9d7d179..75ebefb 100644 --- a/src/remote/operator.rs +++ b/src/remote/operator.rs @@ -3,31 +3,25 @@ use opendal::{services, Operator}; /// Build an [`Operator`] and return `(Operator, relative_path)` from a remote URI. /// -/// Supported schemes: +/// The OpenDAL service set is scoped to forensically-relevant evidence-transfer +/// targets (see ADR-0010). Supported schemes: /// /// **Cloud object storage** -/// `s3`, `gcs`, `azblob`, `azdls`, `azfile`, `b2`, `cos`, `obs`, `oss`, `swift`, `upyun` +/// `s3`, `gcs`, `azblob`, `azdls`, `azfile`, `b2`, `cos`, `obs`, `oss`, `swift` /// -/// **Cloud drives** -/// `gdrive`, `onedrive`, `dropbox`, `aliyun-drive`, `yandex-disk`, `pcloud`, `koofr`, `seafile` +/// **Cloud drive** +/// `gdrive` /// -/// **Developer / ML / infra** -/// `github`, `huggingface`, `vercel-blob`, `vercel-artifacts`, `ghac`, `dbfs` +/// **Hadoop** +/// `webhdfs`, `hdfs` /// -/// **Distributed / big data** -/// `alluxio`, `webhdfs`, `lakefs` -/// -/// **Decentralized** -/// `ipfs`, `ipmfs` -/// -/// **Network KV / databases** -/// `redis`, `memcached`, `etcd`, `tikv`, `mongodb`, `gridfs`, `mysql`, `postgresql`, `sqlite`, -/// `surrealdb`, `cloudflare-kv`, `d1` +/// **SQL** +/// `mysql`, `postgresql`, `sqlite` /// /// **Filesystem / network protocols** /// `file`, `http`, `https`, `webdav`, `sftp`, `ftp`, `ftps` /// -/// **In-memory / embedded** +/// **In-memory** /// `mem` /// /// Auth is read from standard environment variables; refer to each backend's documentation @@ -172,189 +166,7 @@ pub fn operator_for_uri(uri: &str) -> Result<(Operator, String)> { let op = Operator::new(builder)?.finish(); Ok((op, path.to_string())) } - "upyun" => { - // upyun://bucket/key — creds from UPYUN_OPERATOR / UPYUN_PASSWORD - let (bucket, key) = rest.split_once('/').unwrap_or((rest, "")); - let operator_name = std::env::var("UPYUN_OPERATOR").unwrap_or_default(); - let password = std::env::var("UPYUN_PASSWORD").unwrap_or_default(); - let builder = services::Upyun::default() - .bucket(bucket) - .operator(&operator_name) - .password(&password); - let op = Operator::new(builder)?.finish(); - Ok((op, key.to_string())) - } - - // ── Consumer / enterprise cloud drives ─────────────────────────────── - "onedrive" => { - // onedrive://path — token from ONEDRIVE_ACCESS_TOKEN - let token = std::env::var("ONEDRIVE_ACCESS_TOKEN").unwrap_or_default(); - let builder = services::Onedrive::default().root("/").access_token(&token); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "dropbox" => { - // dropbox://path — token from DROPBOX_ACCESS_TOKEN - let token = std::env::var("DROPBOX_ACCESS_TOKEN").unwrap_or_default(); - let builder = services::Dropbox::default().root("/").access_token(&token); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "aliyun-drive" => { - // aliyun-drive://path — token from ALIYUN_DRIVE_ACCESS_TOKEN - let token = std::env::var("ALIYUN_DRIVE_ACCESS_TOKEN").unwrap_or_default(); - let builder = services::AliyunDrive::default() - .root("/") - .access_token(&token); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "yandex-disk" => { - // yandex-disk://path — token from YANDEX_DISK_ACCESS_TOKEN - let token = std::env::var("YANDEX_DISK_ACCESS_TOKEN").unwrap_or_default(); - let builder = services::YandexDisk::default() - .root("/") - .access_token(&token); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "pcloud" => { - // pcloud://path — creds from PCLOUD_USERNAME / PCLOUD_PASSWORD - let endpoint = std::env::var("PCLOUD_ENDPOINT") - .unwrap_or_else(|_| "https://api.pcloud.com".into()); - let username = std::env::var("PCLOUD_USERNAME").unwrap_or_default(); - let password = std::env::var("PCLOUD_PASSWORD").unwrap_or_default(); - let builder = services::Pcloud::default() - .root("/") - .endpoint(&endpoint) - .username(&username) - .password(&password); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "koofr" => { - // koofr://path — creds from KOOFR_EMAIL / KOOFR_PASSWORD - let endpoint = - std::env::var("KOOFR_ENDPOINT").unwrap_or_else(|_| "https://app.koofr.net".into()); - let email = std::env::var("KOOFR_EMAIL").unwrap_or_default(); - let password = std::env::var("KOOFR_PASSWORD").unwrap_or_default(); - let builder = services::Koofr::default() - .root("/") - .endpoint(&endpoint) - .email(&email) - .password(&password); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "seafile" => { - // seafile://server/repo/path — creds from SEAFILE_USERNAME / SEAFILE_PASSWORD - let (server, rest_path) = rest.split_once('/').unwrap_or((rest, "")); - let (repo, path) = rest_path.split_once('/').unwrap_or((rest_path, "")); - let endpoint = format!("https://{server}"); - let username = std::env::var("SEAFILE_USERNAME").unwrap_or_default(); - let password = std::env::var("SEAFILE_PASSWORD").unwrap_or_default(); - let repo_name = if repo.is_empty() { - std::env::var("SEAFILE_REPO").unwrap_or_else(|_| "My Library".into()) - } else { - repo.to_string() - }; - let builder = services::Seafile::default() - .endpoint(&endpoint) - .username(&username) - .password(&password) - .repo_name(&repo_name); - let op = Operator::new(builder)?.finish(); - Ok((op, path.to_string())) - } - - // ── Developer / ML / infra ─────────────────────────────────────────── - "github" => { - // github://owner/repo/path — token from GITHUB_TOKEN - let mut parts = rest.splitn(3, '/'); - let owner = parts.next().unwrap_or(""); - let repo = parts.next().unwrap_or(""); - let path = parts.next().unwrap_or("").to_string(); - let token = std::env::var("GITHUB_TOKEN").unwrap_or_default(); - let builder = services::Github::default() - .token(&token) - .owner(owner) - .repo(repo); - let op = Operator::new(builder)?.finish(); - Ok((op, path)) - } - "huggingface" => { - // huggingface://owner/repo/path — token from HUGGINGFACE_TOKEN - let (repo_id, path) = rest - .split_once('/') - .map(|(a, b)| { - // repo_id is "owner/name", path is the rest - let full = format!("{a}/{b}"); - if let Some(idx) = full.find('/') { - let second = full[idx + 1..].find('/'); - if let Some(second_idx) = second { - let split_at = idx + 1 + second_idx; - ( - full[..split_at].to_string(), - full[split_at + 1..].to_string(), - ) - } else { - (full, String::new()) - } - } else { - (full, String::new()) - } - }) - .unwrap_or((rest.to_string(), String::new())); - let token = std::env::var("HUGGINGFACE_TOKEN").unwrap_or_default(); - let builder = services::Huggingface::default() - .repo_id(&repo_id) - .token(&token); - let op = Operator::new(builder)?.finish(); - Ok((op, path)) - } - "vercel-blob" => { - // vercel-blob://key — token from BLOB_READ_WRITE_TOKEN - let token = std::env::var("BLOB_READ_WRITE_TOKEN").unwrap_or_default(); - let builder = services::VercelBlob::default().token(&token); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "vercel-artifacts" => { - // vercel-artifacts://key — token from VERCEL_ARTIFACTS_TOKEN - let token = std::env::var("VERCEL_ARTIFACTS_TOKEN").unwrap_or_default(); - let builder = services::VercelArtifacts::default().access_token(&token); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "ghac" => { - // ghac://key — for GitHub Actions Cache (useful in CI forensics) - let version = std::env::var("GHAC_VERSION").unwrap_or_else(|_| "v1".into()); - let builder = services::Ghac::default().version(&version); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "dbfs" => { - // dbfs://path — Databricks DBFS; creds from DATABRICKS_HOST + DATABRICKS_TOKEN - let endpoint = std::env::var("DATABRICKS_HOST") - .unwrap_or_else(|_| "https://adb-example.azuredatabricks.net".into()); - let token = std::env::var("DATABRICKS_TOKEN").unwrap_or_default(); - let builder = services::Dbfs::default() - .root("/") - .endpoint(&endpoint) - .token(&token); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - - // ── Distributed / big data ─────────────────────────────────────────── - "alluxio" => { - // alluxio://host:port/path - let (hostport, path) = rest.split_once('/').unwrap_or((rest, "")); - let endpoint = format!("http://{hostport}"); - let builder = services::Alluxio::default().root("/").endpoint(&endpoint); - let op = Operator::new(builder)?.finish(); - Ok((op, path.to_string())) - } + // ── Hadoop ─────────────────────────────────────────────────────────── "webhdfs" => { // webhdfs://host:port/path — user from WEBHDFS_USER let (hostport, path) = rest.split_once('/').unwrap_or((rest, "")); @@ -378,123 +190,7 @@ pub fn operator_for_uri(uri: &str) -> Result<(Operator, String)> { let op = Operator::new(builder)?.finish(); Ok((op, path.to_string())) } - "lakefs" => { - // lakefs://repo/branch/path — creds from LAKEFS_ACCESS_KEY_ID / LAKEFS_SECRET_ACCESS_KEY - let mut parts = rest.splitn(3, '/'); - let repo = parts.next().unwrap_or("").to_string(); - let branch = parts.next().unwrap_or("main").to_string(); - let path = parts.next().unwrap_or("").to_string(); - let endpoint = - std::env::var("LAKEFS_ENDPOINT").unwrap_or_else(|_| "http://localhost:8000".into()); - let username = std::env::var("LAKEFS_ACCESS_KEY_ID").unwrap_or_default(); - let password = std::env::var("LAKEFS_SECRET_ACCESS_KEY").unwrap_or_default(); - let builder = services::Lakefs::default() - .endpoint(&endpoint) - .username(&username) - .password(&password) - .repository(&repo) - .branch(&branch); - let op = Operator::new(builder)?.finish(); - Ok((op, path)) - } - - // ── Decentralized ──────────────────────────────────────────────────── - "ipfs" => { - // ipfs://CID/path — gateway from IPFS_GATEWAY (default: local node) - let gateway = - std::env::var("IPFS_GATEWAY").unwrap_or_else(|_| "http://127.0.0.1:8080".into()); - let builder = services::Ipfs::default().root("/").endpoint(&gateway); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - "ipmfs" => { - // ipmfs:///path — IPFS MFS via local node - let endpoint = - std::env::var("IPFS_ENDPOINT").unwrap_or_else(|_| "http://127.0.0.1:5001".into()); - let builder = services::Ipmfs::default().endpoint(&endpoint); - let op = Operator::new(builder)?.finish(); - Ok((op, rest.to_string())) - } - - // ── Network KV / databases ──────────────────────────────────────────── - #[cfg(feature = "rocksdb-storage")] - "rocksdb" => { - // rocksdb:///path/to/db/key — RocksDB embedded KV - // split_once("://") on "rocksdb:///path/to/db/key" gives rest="/path/to/db/key" - let (db_path, key) = rest.rsplit_once('/').unwrap_or((rest, "")); - let builder = services::Rocksdb::default().datadir(db_path); - let op = Operator::new(builder)?.finish(); - Ok((op, key.to_string())) - } - "rediss" => { - // rediss://host:port/key — Redis with TLS - let (hostport, key) = rest.split_once('/').unwrap_or((rest, "")); - let redis_url = format!("rediss://{hostport}"); - let builder = services::Redis::default().endpoint(&redis_url); - let op = Operator::new(builder)?.finish(); - Ok((op, key.to_string())) - } - "redis" => { - // redis://[user:pass@]host:port/key — path after the host:port is the key - let (conn, path) = rest.split_once('/').unwrap_or((rest, "")); - let redis_url = format!("redis://{conn}"); - let builder = services::Redis::default().endpoint(&redis_url); - let op = Operator::new(builder)?.finish(); - Ok((op, path.to_string())) - } - "memcached" => { - // memcached://host:port/key - let (hostport, path) = rest.split_once('/').unwrap_or((rest, "")); - let endpoint = format!("tcp://{hostport}"); - let builder = services::Memcached::default().endpoint(&endpoint); - let op = Operator::new(builder)?.finish(); - Ok((op, path.to_string())) - } - "etcd" => { - // etcd://host:port/key - let (hostport, path) = rest.split_once('/').unwrap_or((rest, "")); - let endpoint = format!("http://{hostport}"); - let builder = services::Etcd::default().endpoints(&endpoint); - let op = Operator::new(builder)?.finish(); - Ok((op, path.to_string())) - } - "tikv" => { - // tikv://pd-host:port/key - let (hostport, path) = rest.split_once('/').unwrap_or((rest, "")); - let builder = services::Tikv::default().endpoints(vec![hostport.to_string()]); - let op = Operator::new(builder)?.finish(); - Ok((op, path.to_string())) - } - "mongodb" => { - // mongodb://[user:pass@]host/db/collection/key - let conn_str = format!("mongodb://{rest}"); - let mut parts = rest.splitn(3, '/'); - let _ = parts.next(); // host - let database = parts.next().unwrap_or("blazehash"); - let rest_path = parts.next().unwrap_or(""); - let (collection, path) = rest_path.split_once('/').unwrap_or((rest_path, "")); - let builder = services::Mongodb::default() - .connection_string(&conn_str) - .database(database) - .collection(collection); - let op = Operator::new(builder)?.finish(); - Ok((op, path.to_string())) - } - "gridfs" => { - // gridfs://[user:pass@]host/db/bucket/key - let conn_str = format!("mongodb://{rest}"); - let mut parts = rest.splitn(3, '/'); - let _ = parts.next(); // host[:port] - let database = parts.next().unwrap_or("blazehash"); - let rest_path = parts.next().unwrap_or(""); - let (bucket, path) = rest_path.split_once('/').unwrap_or((rest_path, "")); - let builder = services::Gridfs::default() - .connection_string(&conn_str) - .database(database) - .bucket(bucket); - let op = Operator::new(builder)?.finish(); - Ok((op, path.to_string())) - } + // ── SQL (sqlx, pure Rust) ───────────────────────────────────────────── "mysql" => { // mysql://[user:pass@]host/db/key let conn_str = format!("mysql://{rest}"); @@ -526,32 +222,6 @@ pub fn operator_for_uri(uri: &str) -> Result<(Operator, String)> { let op = Operator::new(builder)?.finish(); Ok((op, key.to_string())) } - // "surrealdb" — not wired: pulls async-graphql@7.2.1 (requires rustc 1.89 > our MSRV 1.85) - "cloudflare-kv" => { - // cloudflare-kv://namespace-id/key - let (namespace, key) = rest.split_once('/').unwrap_or((rest, "")); - let account_id = std::env::var("CLOUDFLARE_ACCOUNT_ID").unwrap_or_default(); - let token = std::env::var("CLOUDFLARE_API_TOKEN").unwrap_or_default(); - let builder = services::CloudflareKv::default() - .account_id(&account_id) - .api_token(&token) - .namespace_id(namespace); - let op = Operator::new(builder)?.finish(); - Ok((op, key.to_string())) - } - "d1" => { - // d1://database-id/key — Cloudflare D1 (SQLite via REST) - let (db_id, key) = rest.split_once('/').unwrap_or((rest, "")); - let account_id = std::env::var("CLOUDFLARE_ACCOUNT_ID").unwrap_or_default(); - let token = std::env::var("CLOUDFLARE_API_TOKEN").unwrap_or_default(); - let builder = services::D1::default() - .account_id(&account_id) - .token(&token) - .database_id(db_id); - let op = Operator::new(builder)?.finish(); - Ok((op, key.to_string())) - } - // ── Filesystem / network protocols ─────────────────────────────────── "webdav" => { let host = rest.split('/').next().unwrap_or(""); @@ -588,31 +258,6 @@ pub fn operator_for_uri(uri: &str) -> Result<(Operator, String)> { crate::remote::ftp::fetch_ftp_bytes(), not operator_for_uri()" ) } - "monoiofs" => { - // monoiofs:///abs/path — monoio-based async fs (Linux io_uring only) - let full = format!("/{rest}"); - let (dir, file) = full.rsplit_once('/').unwrap_or(("/", &full)); - #[cfg(target_os = "linux")] - { - let builder = services::Monoiofs::default().root(dir); - let op = Operator::new(builder)?.finish(); - Ok((op, file.to_string())) - } - #[cfg(not(target_os = "linux"))] - { - let _ = (dir, file); - bail!("monoiofs:// is only supported on Linux (io_uring required)") - } - } - "compfs" => { - // compfs:///abs/path/to/dir — compio-based async filesystem - // rest = "/abs/path/to/dir/file" → root="/abs/path/to/dir", path="file" - let full = format!("/{rest}"); // restore leading slash stripped by split_once("://") - let (dir, file) = full.rsplit_once('/').unwrap_or(("/", &full)); - let builder = services::Compfs::default().root(dir); - let op = Operator::new(builder)?.finish(); - Ok((op, file.to_string())) - } "file" => { let (dir, file) = rest.rsplit_once('/').unwrap_or(("/", rest)); let builder = services::Fs::default().root(dir); @@ -678,4 +323,57 @@ mod tests { "webhdfs:// should be supported" ); } + + // ── ADR-0010: dropped OpenDAL backends must be unsupported ──────────────── + // These schemes were removed from the `remote` OpenDAL service set. Each must + // fall through to the `other =>` arm and report "unsupported URI scheme". + fn assert_unsupported(uri: &str) { + let err = operator_for_uri(uri) + .expect_err("dropped scheme must return an error") + .to_string(); + assert!( + err.contains("unsupported URI scheme"), + "expected unsupported-scheme error for {uri:?}, got: {err}" + ); + } + + #[test] + fn dropped_mongodb_is_unsupported() { + assert_unsupported("mongodb://host/db/coll/key"); + } + + #[test] + fn dropped_redis_is_unsupported() { + assert_unsupported("redis://host:6379/key"); + } + + #[test] + fn dropped_rocksdb_is_unsupported() { + assert_unsupported("rocksdb:///var/lib/db/key"); + } + + #[test] + fn dropped_ipfs_is_unsupported() { + assert_unsupported("ipfs://QmHash/path"); + } + + #[test] + fn dropped_onedrive_is_unsupported() { + assert_unsupported("onedrive://Documents/file.pdf"); + } + + #[test] + fn dropped_github_is_unsupported() { + assert_unsupported("github://owner/repo/path"); + } + + #[test] + fn dropped_compfs_is_unsupported() { + assert_unsupported("compfs:///tmp/evidence/file"); + } + + #[test] + fn dropped_etcd_is_unsupported() { + assert_unsupported("etcd://host:2379/key"); + } } diff --git a/tests/remote_uri_tests.rs b/tests/remote_uri_tests.rs index db58c0f..2f029bb 100644 --- a/tests/remote_uri_tests.rs +++ b/tests/remote_uri_tests.rs @@ -411,23 +411,6 @@ fn scheme_detect_vercel_blob() { ); } -// ── RocksDB (optional feature: rocksdb-storage) ─────────────────────────────── - -#[cfg(feature = "rocksdb-storage")] -#[test] -fn rocksdb_uri_is_detected() { - assert!(is_remote_uri("rocksdb:///var/lib/rocksdb/evidence")); -} - -#[cfg(feature = "rocksdb-storage")] -#[test] -fn scheme_detect_rocksdb() { - assert_eq!( - UriScheme::detect("rocksdb:///path/to/db"), - Some(UriScheme::RocksDb) - ); -} - // ── Monoio filesystem (monoiofs, Linux only) ────────────────────────────────── #[cfg(target_os = "linux")] @@ -512,16 +495,6 @@ mod operator_tests { ); } - #[test] - fn operator_onedrive_not_unsupported() { - assert_not_unsupported("onedrive://Documents/file.pdf"); - } - - #[test] - fn operator_dropbox_not_unsupported() { - assert_not_unsupported("dropbox://path/file"); - } - #[test] fn operator_b2_not_unsupported() { assert_not_unsupported("b2://bucket/key"); @@ -557,76 +530,11 @@ mod operator_tests { assert_not_unsupported("azdls://container/path"); } - #[test] - fn operator_github_not_unsupported() { - assert_not_unsupported("github://owner/repo/path"); - } - - #[test] - fn operator_ipfs_not_unsupported() { - assert_not_unsupported("ipfs://QmHash/path"); - } - - #[test] - fn operator_ipmfs_not_unsupported() { - assert_not_unsupported("ipmfs:///path"); - } - #[test] fn operator_webhdfs_not_unsupported() { assert_not_unsupported("webhdfs://host:50070/path"); } - #[test] - fn operator_alluxio_not_unsupported() { - assert_not_unsupported("alluxio://master:19998/path"); - } - - #[test] - fn operator_lakefs_not_unsupported() { - assert_not_unsupported("lakefs://repo/main/file"); - } - - #[test] - fn operator_seafile_not_unsupported() { - assert_not_unsupported("seafile://server/repo/file"); - } - - #[test] - fn operator_pcloud_not_unsupported() { - assert_not_unsupported("pcloud://path/file"); - } - - #[test] - fn operator_koofr_not_unsupported() { - assert_not_unsupported("koofr://path/file"); - } - - #[test] - fn operator_yandex_disk_not_unsupported() { - assert_not_unsupported("yandex-disk://path/file"); - } - - #[test] - fn operator_huggingface_not_unsupported() { - assert_not_unsupported("huggingface://owner/repo/file"); - } - - #[test] - fn operator_upyun_not_unsupported() { - assert_not_unsupported("upyun://bucket/key"); - } - - #[test] - fn operator_aliyun_drive_not_unsupported() { - assert_not_unsupported("aliyun-drive://path/file"); - } - - #[test] - fn operator_vercel_blob_not_unsupported() { - assert_not_unsupported("vercel-blob://key"); - } - // ── sftp path extraction (via parse_sftp_uri — cross-platform ssh2/libssh2) ── // operator_for_uri bails directing callers to fetch_sftp_bytes(); parse_sftp_uri // is the path-extraction API for SFTP, same pattern as parse_ftp_uri for FTP. @@ -683,67 +591,6 @@ mod operator_tests { ); } - // ── rocksdb:// (optional feature: rocksdb-storage) ──────────────────────── - - #[cfg(feature = "rocksdb-storage")] - #[test] - fn operator_rocksdb_not_unsupported() { - assert_not_unsupported("rocksdb:///tmp/bh-rocksdb-test-a/evidence/manifest"); - } - - #[cfg(feature = "rocksdb-storage")] - #[test] - fn rocksdb_path_extracted_correctly() { - let (_, key) = operator_for_uri("rocksdb:///tmp/bh-rocksdb-test-b/evidence/case-001") - .expect("rocksdb:// should be supported with rocksdb-storage feature"); - assert_eq!(key, "case-001"); - } - - // ── monoiofs:// (monoio filesystem, Linux only) ─────────────────────────── - - #[cfg(target_os = "linux")] - #[test] - fn operator_monoiofs_not_unsupported() { - assert_not_unsupported("monoiofs:///tmp/evidence"); - } - - #[cfg(target_os = "linux")] - #[test] - fn monoiofs_path_extracted_correctly() { - // Use /tmp so the operator can create its root dir without needing root - let (_, path) = operator_for_uri("monoiofs:///tmp/blazehash-test-monoiofs/disk.dd") - .expect("monoiofs:// should be supported"); - assert_eq!(path, "disk.dd"); - } - - // ── compfs:// (compio filesystem) ───────────────────────────────────────── - - #[test] - fn operator_compfs_not_unsupported() { - assert_not_unsupported("compfs:///tmp/evidence"); - } - - #[test] - fn compfs_path_extracted_correctly() { - let (_, path) = operator_for_uri("compfs:///tmp/evidence/disk.dd") - .expect("compfs:// should be supported"); - assert_eq!(path, "disk.dd"); - } - - // ── rediss:// (Redis with TLS) ───────────────────────────────────────────── - - #[test] - fn operator_rediss_not_unsupported() { - assert_not_unsupported("rediss://host:6380/mykey"); - } - - #[test] - fn rediss_path_extracted_correctly() { - let (_, key) = operator_for_uri("rediss://localhost:6380/cache:evidence:hash") - .expect("rediss:// should be supported"); - assert_eq!(key, "cache:evidence:hash"); - } - // ── HDFS native path extraction (TDD) ───────────────────────────────────── #[test]