From a51f3b005949b0a2c02c8cb44aca04dd1fc2cc34 Mon Sep 17 00:00:00 2001 From: Matthew Turner Date: Tue, 1 Sep 2026 12:11:39 -0400 Subject: [PATCH] Functions rocksdb --- .github/workflows/test.yml | 4 ++-- CLAUDE.md | 8 +++---- Cargo.lock | 24 +++++++++---------- Cargo.toml | 7 ++++-- README.md | 2 +- crates/datafusion-app/Cargo.toml | 4 ++-- crates/datafusion-app/src/local.rs | 8 +++---- .../Cargo.toml | 6 ++--- .../README.md | 0 .../src/cf_metrics.rs | 0 .../src/lib.rs | 0 .../src/metadata.rs | 0 .../src/sstables.rs | 0 docs/features.md | 4 ++-- tests/extension_cases/mod.rs | 2 +- tests/extension_cases/rocksdb.rs | 4 ++-- 16 files changed, 38 insertions(+), 35 deletions(-) rename crates/{datafusion-rocksdb => datafusion-functions-rocksdb}/Cargo.toml (82%) rename crates/{datafusion-rocksdb => datafusion-functions-rocksdb}/README.md (100%) rename crates/{datafusion-rocksdb => datafusion-functions-rocksdb}/src/cf_metrics.rs (100%) rename crates/{datafusion-rocksdb => datafusion-functions-rocksdb}/src/lib.rs (100%) rename crates/{datafusion-rocksdb => datafusion-functions-rocksdb}/src/metadata.rs (100%) rename crates/{datafusion-rocksdb => datafusion-functions-rocksdb}/src/sstables.rs (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index afb788a..536c09c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -353,8 +353,8 @@ jobs: uses: ./.github/actions/setup-rust - name: Run RocksDB tests run: | - cargo test --manifest-path crates/datafusion-rocksdb/Cargo.toml - cargo test --features=rocksdb extension_cases::rocksdb + cargo test --manifest-path crates/datafusion-functions-rocksdb/Cargo.toml + cargo test --features=functions-rocksdb extension_cases::rocksdb test-net: name: Extension / Net runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index c5bf618..7ba123f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -111,7 +111,7 @@ cargo test --features=udfs-wasm extension_cases::udfs_wasm cargo test --features=vortex extension_cases::vortex cargo test --features=vortex cli_cases::basic::test_output_vortex cargo test --features=net extension_cases::net # Requires libpcap (libpcap-dev on Debian/Ubuntu) -cargo test --features=rocksdb extension_cases::rocksdb +cargo test --features=functions-rocksdb extension_cases::rocksdb # Run tests for specific crates cargo test --manifest-path crates/datafusion-app/Cargo.toml --all-features @@ -119,7 +119,7 @@ cargo test --manifest-path crates/datafusion-functions-parquet/Cargo.toml cargo test --manifest-path crates/datafusion-functions-arrow/Cargo.toml cargo test --manifest-path crates/datafusion-udfs-wasm/Cargo.toml cargo test --manifest-path crates/datafusion-net/Cargo.toml --all-features -cargo test --manifest-path crates/datafusion-rocksdb/Cargo.toml +cargo test --manifest-path crates/datafusion-functions-rocksdb/Cargo.toml # Run a single test cargo test @@ -177,7 +177,7 @@ The project is organized as a workspace with multiple crates: - **`crates/datafusion-net`**: Network packet capture querying — `pcap` (capture files), `capture` (live capture), `interfaces` (list capture devices), and `tcp_conversations` (per-connection TCP flow analytics) table functions (`capture`/`interfaces` behind its `live` feature); the `reverse_dns` (IP → hostname via PTR lookup) and `geoip` (IP → location via MaxMind `.mmdb` database) scalar UDFs; the `dns_query` and `tls_sni` payload-decoding scalar UDFs; and `pcap_wide` / `capture_wide` variants that append DNS and geolocation enrichment columns -- **`crates/datafusion-rocksdb`**: RocksDB database inspection table functions — `rocksdb_metadata` (DB-level summary), `rocksdb_sstables` (one row per live SST file), and `rocksdb_cf_metrics` (per-column-family RocksDB properties in long format); databases are opened read-only so live databases can be inspected +- **`crates/datafusion-functions-rocksdb`**: RocksDB database inspection table functions — `rocksdb_metadata` (DB-level summary), `rocksdb_sstables` (one row per live SST file), and `rocksdb_cf_metrics` (per-column-family RocksDB properties in long format); databases are opened read-only so live databases can be inspected - **`crates/datafusion-auth`**: Authentication implementations @@ -217,6 +217,7 @@ The project uses extensive feature flags to keep binary size manageable: - `functions-parquet` - Parquet-specific functions (default) - `functions-arrow` - Arrow IPC file inspection functions - `functions-json` - JSON functions +- `functions-rocksdb` - Inspect RocksDB databases (metadata, SST files, column family metrics) via SQL - `clickhouse` - Register ClickHouse instances as catalogs - `mongodb` - Register MongoDB instances as catalogs - `deltalake` - Delta Lake table format support @@ -225,7 +226,6 @@ The project uses extensive feature flags to keep binary size manageable: - `http` - HTTP server - `huggingface` - HuggingFace dataset integration - `net` - Query network packet captures (pcap files and live capture) via SQL -- `rocksdb` - Inspect RocksDB databases (metadata, SST files, column family metrics) via SQL - `websocket` - WebSocket table function for streaming WebSocket messages as rows - `udfs-wasm` - WASM UDF support - `observability` - Metrics and tracing (required by servers) diff --git a/Cargo.lock b/Cargo.lock index 7ce4897..c5dea0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2654,8 +2654,8 @@ dependencies = [ "datafusion-functions-arrow", "datafusion-functions-json", "datafusion-functions-parquet", + "datafusion-functions-rocksdb", "datafusion-net", - "datafusion-rocksdb", "datafusion-table-providers", "datafusion-udfs-wasm", "deltalake", @@ -2915,8 +2915,8 @@ dependencies = [ "crossterm", "datafusion", "datafusion-app", + "datafusion-functions-rocksdb", "datafusion-net", - "datafusion-rocksdb", "datafusion-udfs-wasm", "directories", "env_logger", @@ -3167,6 +3167,16 @@ dependencies = [ "parquet", ] +[[package]] +name = "datafusion-functions-rocksdb" +version = "0.1.0" +dependencies = [ + "arrow", + "async-trait", + "datafusion", + "rocksdb", +] + [[package]] name = "datafusion-functions-table" version = "54.1.0" @@ -3415,16 +3425,6 @@ dependencies = [ "log", ] -[[package]] -name = "datafusion-rocksdb" -version = "0.1.0" -dependencies = [ - "arrow", - "async-trait", - "datafusion", - "rocksdb", -] - [[package]] name = "datafusion-session" version = "54.1.0" diff --git a/Cargo.toml b/Cargo.toml index 74b26de..cb10db8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ datafusion = { version = "54" } datafusion-app = { path = "crates/datafusion-app", version = "0.1.0" } # Optional (not a dev-dependency) so that plain `cargo test` does not compile # librocksdb; used from feature-gated test code to generate fixture databases. -datafusion-rocksdb = { optional = true, path = "crates/datafusion-rocksdb", version = "0.1.0" } +datafusion-functions-rocksdb = { optional = true, path = "crates/datafusion-functions-rocksdb", version = "0.1.0" } directories = "5.0.1" env_logger = "0.11.5" futures = "0.3.30" @@ -113,6 +113,10 @@ flightsql = [ functions-arrow = ["datafusion-app/functions-arrow"] functions-json = ["datafusion-app/functions-json"] functions-parquet = ["datafusion-app/functions-parquet"] +functions-rocksdb = [ + "datafusion-app/functions-rocksdb", + "dep:datafusion-functions-rocksdb", +] http = [ "axum", "datafusion-app/observability", @@ -125,7 +129,6 @@ http = [ huggingface = ["datafusion-app/huggingface"] mongodb = ["datafusion-app/mongodb"] net = ["datafusion-app/net"] -rocksdb = ["datafusion-app/rocksdb", "dep:datafusion-rocksdb"] s3 = ["datafusion-app/s3"] tui = ["dep:crossterm", "dep:ratatui", "dep:ratatui-textarea", "dep:tui-logger"] udfs-wasm = ["datafusion-app/udfs-wasm"] diff --git a/README.md b/README.md index ed86d1c..6c31d1c 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ cargo install datafusion-dft --features=websocket cargo install datafusion-dft --features=net # With the rocksdb_* table functions for inspecting RocksDB databases -cargo install datafusion-dft --features=rocksdb +cargo install datafusion-dft --features=functions-rocksdb # With Vortex file format support cargo install datafusion-dft --features=vortex diff --git a/crates/datafusion-app/Cargo.toml b/crates/datafusion-app/Cargo.toml index 48cb80f..1f455d7 100644 --- a/crates/datafusion-app/Cargo.toml +++ b/crates/datafusion-app/Cargo.toml @@ -16,8 +16,8 @@ datafusion = "54" datafusion-functions-arrow = { optional = true, path = "../datafusion-functions-arrow", version = "0.1.0" } datafusion-functions-json = { optional = true, version = "0.54" } datafusion-functions-parquet = { optional = true, path = "../datafusion-functions-parquet", version = "0.1.0" } +datafusion-functions-rocksdb = { optional = true, path = "../datafusion-functions-rocksdb", version = "0.1.0" } datafusion-net = { optional = true, path = "../datafusion-net", version = "0.1.0" } -datafusion-rocksdb = { optional = true, path = "../datafusion-rocksdb", version = "0.1.0" } datafusion-table-providers = { optional = true, version = "0.13.0" } datafusion-udfs-wasm = { features = [ "serde", @@ -74,6 +74,7 @@ flightsql = ["dep:arrow-flight", "dep:base64", "dep:tonic"] functions-arrow = ["dep:datafusion-functions-arrow"] functions-json = ["dep:datafusion-functions-json"] functions-parquet = ["dep:datafusion-functions-parquet"] +functions-rocksdb = ["dep:datafusion-functions-rocksdb"] huggingface = ["object_store_opendal", "opendal", "url"] mongodb = [ "datafusion-table-providers/mongodb", @@ -81,7 +82,6 @@ mongodb = [ ] net = ["datafusion-net/live", "dep:datafusion-net"] observability = ["dep:metrics", "dep:tokio-metrics"] -rocksdb = ["dep:datafusion-rocksdb"] s3 = ["object_store/aws", "url"] udfs-wasm = ["dep:datafusion-udfs-wasm"] vortex = ["dep:vortex-datafusion"] diff --git a/crates/datafusion-app/src/local.rs b/crates/datafusion-app/src/local.rs index 2fd0324..77ace6c 100644 --- a/crates/datafusion-app/src/local.rs +++ b/crates/datafusion-app/src/local.rs @@ -231,19 +231,19 @@ impl ExecutionContext { ); } - #[cfg(feature = "rocksdb")] + #[cfg(feature = "functions-rocksdb")] { session_ctx.register_udtf( "rocksdb_metadata", - Arc::new(datafusion_rocksdb::RocksDbMetadataFunc {}), + Arc::new(datafusion_functions_rocksdb::RocksDbMetadataFunc {}), ); session_ctx.register_udtf( "rocksdb_sstables", - Arc::new(datafusion_rocksdb::RocksDbSstablesFunc {}), + Arc::new(datafusion_functions_rocksdb::RocksDbSstablesFunc {}), ); session_ctx.register_udtf( "rocksdb_cf_metrics", - Arc::new(datafusion_rocksdb::RocksDbCfMetricsFunc {}), + Arc::new(datafusion_functions_rocksdb::RocksDbCfMetricsFunc {}), ); } diff --git a/crates/datafusion-rocksdb/Cargo.toml b/crates/datafusion-functions-rocksdb/Cargo.toml similarity index 82% rename from crates/datafusion-rocksdb/Cargo.toml rename to crates/datafusion-functions-rocksdb/Cargo.toml index 27cb10d..b063551 100644 --- a/crates/datafusion-rocksdb/Cargo.toml +++ b/crates/datafusion-functions-rocksdb/Cargo.toml @@ -2,12 +2,12 @@ authors = ["Matthew Turner "] description = "DataFusion table functions for inspecting RocksDB databases" edition = "2021" -homepage = "https://github.com/datafusion-contrib/datafusion-dft/tree/main/crates/datafusion-rocksdb" +homepage = "https://github.com/datafusion-contrib/datafusion-dft/tree/main/crates/datafusion-functions-rocksdb" keywords = ["datafusion", "query", "rocksdb", "sql"] license = "Apache-2.0" -name = "datafusion-rocksdb" +name = "datafusion-functions-rocksdb" readme = "README.md" -repository = "https://github.com/datafusion-contrib/datafusion-dft/tree/main/crates/datafusion-rocksdb" +repository = "https://github.com/datafusion-contrib/datafusion-dft/tree/main/crates/datafusion-functions-rocksdb" version = "0.1.0" [dependencies] diff --git a/crates/datafusion-rocksdb/README.md b/crates/datafusion-functions-rocksdb/README.md similarity index 100% rename from crates/datafusion-rocksdb/README.md rename to crates/datafusion-functions-rocksdb/README.md diff --git a/crates/datafusion-rocksdb/src/cf_metrics.rs b/crates/datafusion-functions-rocksdb/src/cf_metrics.rs similarity index 100% rename from crates/datafusion-rocksdb/src/cf_metrics.rs rename to crates/datafusion-functions-rocksdb/src/cf_metrics.rs diff --git a/crates/datafusion-rocksdb/src/lib.rs b/crates/datafusion-functions-rocksdb/src/lib.rs similarity index 100% rename from crates/datafusion-rocksdb/src/lib.rs rename to crates/datafusion-functions-rocksdb/src/lib.rs diff --git a/crates/datafusion-rocksdb/src/metadata.rs b/crates/datafusion-functions-rocksdb/src/metadata.rs similarity index 100% rename from crates/datafusion-rocksdb/src/metadata.rs rename to crates/datafusion-functions-rocksdb/src/metadata.rs diff --git a/crates/datafusion-rocksdb/src/sstables.rs b/crates/datafusion-functions-rocksdb/src/sstables.rs similarity index 100% rename from crates/datafusion-rocksdb/src/sstables.rs rename to crates/datafusion-functions-rocksdb/src/sstables.rs diff --git a/docs/features.md b/docs/features.md index 687b2b0..2a4a53f 100644 --- a/docs/features.md +++ b/docs/features.md @@ -237,9 +237,9 @@ SELECT tls_sni(payload) AS host, count(*) AS hellos FROM pcap('capture.pcap') WHERE tls_sni(payload) IS NOT NULL GROUP BY host ORDER BY hellos DESC ``` -### RocksDB (`--features=rocksdb`) +### RocksDB (`--features=functions-rocksdb`) -Adds table functions from [datafusion-rocksdb](https://github.com/datafusion-contrib/datafusion-dft/tree/main/crates/datafusion-rocksdb) for inspecting RocksDB databases with SQL. Databases are opened read-only and the LOCK file is never taken, so a database another process has open read-write can be inspected safely. Data still in the WAL is not visible to a read-only handle: memtable-related metrics read 0 and key estimates exclude unflushed writes. +Adds table functions from [datafusion-functions-rocksdb](https://github.com/datafusion-contrib/datafusion-dft/tree/main/crates/datafusion-functions-rocksdb) for inspecting RocksDB databases with SQL. Databases are opened read-only and the LOCK file is never taken, so a database another process has open read-write can be inspected safely. Data still in the WAL is not visible to a read-only handle: memtable-related metrics read 0 and key estimates exclude unflushed writes. `rocksdb_metadata` returns a single database-level summary row: column families, latest sequence number, live SST file count and total size, estimated key count, snapshot count, and MANIFEST / WAL file details: diff --git a/tests/extension_cases/mod.rs b/tests/extension_cases/mod.rs index 523e528..30b7b00 100644 --- a/tests/extension_cases/mod.rs +++ b/tests/extension_cases/mod.rs @@ -37,7 +37,7 @@ mod huggingface; mod mongodb; #[cfg(feature = "net")] mod net; -#[cfg(feature = "rocksdb")] +#[cfg(feature = "functions-rocksdb")] mod rocksdb; #[cfg(feature = "s3")] mod s3; diff --git a/tests/extension_cases/rocksdb.rs b/tests/extension_cases/rocksdb.rs index b332f1d..e5d575a 100644 --- a/tests/extension_cases/rocksdb.rs +++ b/tests/extension_cases/rocksdb.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -//! Tests for datafusion-rocksdb integration +//! Tests for datafusion-functions-rocksdb integration -use datafusion_rocksdb::rocksdb::{Options, DB}; +use datafusion_functions_rocksdb::rocksdb::{Options, DB}; use crate::extension_cases::TestExecution;