From a1bca9adea05ab8d98ed2d1cafe888a095ec0887 Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Tue, 15 Apr 2025 15:55:59 +0200 Subject: [PATCH 01/33] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f2eaa189..44c47554 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ A very specialized timeseries database created for events, logs, traces and metr Its designed to allow users plug in their own s3 storage and buckets and have their stored to their accounts. This way, timefusion is used as a compute and cache engine, not primary data storage. +Timefusion speaks the postgres dialect, so you can insert and read from it using any postgres client or driver. + ## Configuration Timefusion can be configured using the following environment variables: From 3ff06fea67bc91bdeef1f5ebaca5896598d146b9 Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Wed, 16 Apr 2025 00:43:36 +0200 Subject: [PATCH 02/33] add support for date based partitioning --- Cargo.lock | 398 +++++++++++++++++++++++++------------- Cargo.toml | 6 +- src/database.rs | 123 ++++++------ src/persistent_queue.rs | 53 ++++- tests/integration_test.rs | 12 +- 5 files changed, 374 insertions(+), 218 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60d20a39..fd140127 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -366,9 +366,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "array-init" @@ -442,9 +442,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "54.3.0" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6ed265c73f134a583d02c3cab5e16afab9446d8048ede8707e31f85fad58a0" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" dependencies = [ "bytes", "half", @@ -490,9 +490,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "54.3.0" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f2cebf504bb6a92a134a87fff98f01b14fbb3a93ecf7aef90cd0f888c5fffa4" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" dependencies = [ "arrow-buffer", "arrow-schema", @@ -527,7 +527,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap", + "indexmap 2.9.0", "lexical-core", "num", "serde", @@ -562,9 +562,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "54.3.0" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c53775bba63f319189f366d2b86e9a8889373eb198f07d8544938fc9f8ed9a" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" dependencies = [ "bitflags 2.9.0", "serde", @@ -694,9 +694,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.12.6" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -705,9 +705,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f" +checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" dependencies = [ "bindgen", "cc", @@ -744,9 +744,9 @@ dependencies = [ [[package]] name = "aws-sdk-dynamodb" -version = "1.70.0" +version = "1.71.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac281113af7f8700394bf25eb272b842b7ca088810e96c928f812282f2e6f44" +checksum = "2d49d08b1c99ca9a7de728a8975504857f2c24581a177f952e2a10244c305a1c" dependencies = [ "aws-credential-types", "aws-runtime", @@ -802,9 +802,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.63.0" +version = "1.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1cb45b83b53b5cd55ee33fd9fd8a70750255a3f286e4dca20e882052f2b256f" +checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" dependencies = [ "aws-credential-types", "aws-runtime", @@ -825,9 +825,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.64.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4d9bc075ea6238778ed3951b65d3cde8c3864282d64fdcd19f2a90c0609f1" +checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" dependencies = [ "aws-credential-types", "aws-runtime", @@ -848,9 +848,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.64.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819ccba087f403890fee4825eeab460e64c59345667d2b83a12cf544b581e3a7" +checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" dependencies = [ "aws-credential-types", "aws-runtime", @@ -974,7 +974,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.4.8", + "h2 0.4.9", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", @@ -985,7 +985,7 @@ dependencies = [ "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -1115,9 +1115,9 @@ dependencies = [ [[package]] name = "backon" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "970d91570c01a8a5959b36ad7dd1c30642df24b6b3068710066f6809f7033bb7" +checksum = "fd0b50b1b78dbadd44ab18b3c794e496f3a139abb9fbc27d9c94c4eebbb96496" dependencies = [ "fastrand", "tokio", @@ -1187,9 +1187,9 @@ dependencies = [ [[package]] name = "bigdecimal" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c" +checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" dependencies = [ "autocfg", "libm", @@ -1274,9 +1274,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17679a8d69b6d7fd9cd9801a536cec9fa5e5970b69f9d4747f70b39b031f5e7" +checksum = "389a099b34312839e16420d499a9cad9650541715937ffbdd40d36f49e77eeb3" dependencies = [ "arrayref", "arrayvec", @@ -1454,9 +1454,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.17" +version = "1.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" dependencies = [ "jobserver", "libc", @@ -1570,9 +1570,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.34" +version = "4.5.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e958897981290da2a852763fe9cdb89cd36977a5d729023127095fa94d95e2ff" +checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04" dependencies = [ "clap_builder", "clap_derive", @@ -1580,9 +1580,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.34" +version = "4.5.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b0f35019843db2160b5bb19ae09b4e6411ac33fc6a712003c33e03090e2489" +checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5" dependencies = [ "anstream", "anstyle", @@ -1917,6 +1917,41 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.100", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.100", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -2036,7 +2071,7 @@ dependencies = [ "base64 0.22.1", "half", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.9.0", "libc", "log 0.4.27", "object_store", @@ -2131,7 +2166,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap", + "indexmap 2.9.0", "paste", "recursive", "serde_json", @@ -2146,7 +2181,7 @@ checksum = "18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e" dependencies = [ "arrow", "datafusion-common", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "paste", ] @@ -2312,7 +2347,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "log 0.4.27", "recursive", @@ -2335,7 +2370,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "log 0.4.27", "paste", @@ -2397,7 +2432,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "log 0.4.27", "parking_lot 0.12.3", @@ -2408,7 +2443,7 @@ dependencies = [ [[package]] name = "datafusion-postgres" version = "0.3.0" -source = "git+https://github.com/apitoolkit/datafusion-postgres.git?branch=insert-query-compliance#a5ad9c9fd523e6a52b3fabd0889e0d612b8e3cd3" +source = "git+https://github.com/sunng87/datafusion-postgres.git#2cf58787a8bf3e12a82b836d7dbdc5f6aee9f5a6" dependencies = [ "async-trait", "chrono", @@ -2454,7 +2489,7 @@ dependencies = [ "bigdecimal", "datafusion-common", "datafusion-expr", - "indexmap", + "indexmap 2.9.0", "log 0.4.27", "recursive", "regex 1.11.1", @@ -2463,8 +2498,8 @@ dependencies = [ [[package]] name = "datafusion-uwheel" -version = "40.0.0" -source = "git+https://github.com/apitoolkit/datafusion-uwheel.git?branch=datafusion-46#7d7d1223470f06d1ae8732929f40d61e8698ec57" +version = "46.0.0" +source = "git+https://github.com/apitoolkit/datafusion-uwheel.git?branch=datafusion-46#053dada166281ab3030ec9edcaaf39af741d09bb" dependencies = [ "bitpacking", "chrono", @@ -2485,7 +2520,7 @@ dependencies = [ "fix-hidden-lifetime-bug", "futures", "home", - "indexmap", + "indexmap 2.9.0", "itertools 0.13.0", "object_store", "parquet", @@ -2589,7 +2624,7 @@ dependencies = [ "fix-hidden-lifetime-bug", "futures", "humantime", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "libc", "maplit", @@ -2629,11 +2664,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -2806,9 +2842,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" dependencies = [ "anstream", "anstyle", @@ -2825,9 +2861,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", "windows-sys 0.59.0", @@ -2909,12 +2945,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "miniz_oxide 0.8.5", + "miniz_oxide 0.8.8", ] [[package]] @@ -3154,7 +3190,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", @@ -3163,9 +3199,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" dependencies = [ "atomic-waker", "bytes", @@ -3173,7 +3209,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", @@ -3182,9 +3218,9 @@ dependencies = [ [[package]] name = "half" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "bytemuck", "cfg-if", @@ -3377,7 +3413,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.8", + "h2 0.4.9", "http 1.3.1", "http-body 1.0.1", "httparse", @@ -3414,7 +3450,7 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -3440,9 +3476,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-channel", @@ -3450,6 +3486,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "hyper 1.6.0", + "libc", "pin-project-lite", "socket2", "tokio", @@ -3459,9 +3496,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.62" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3599,6 +3636,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -3634,12 +3677,24 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "indexmap" -version = "2.8.0" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown 0.15.2", + "serde", ] [[package]] @@ -3739,9 +3794,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260" +checksum = "e5ad87c89110f55e4cd4dc2893a9790820206729eaf221555f742d540b0724a0" dependencies = [ "jiff-static", "log 0.4.27", @@ -3752,9 +3807,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" +checksum = "d076d5b64a7e2fe6f0743f02c43ca4a6725c0f904203bfe276a5b3e793103605" dependencies = [ "proc-macro2", "quote", @@ -3778,10 +3833,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.2", "libc", ] @@ -3902,9 +3958,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.171" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libloading" @@ -3942,9 +3998,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" @@ -4031,9 +4087,9 @@ checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" [[package]] name = "marrow" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd5fc5916496c19f17c6b7b6ebc8210546f3fe42d59179efe78ac9562e09a2d6" +checksum = "3641f6a55539a8b6e5349b3bdfb5b315714fbceda3253815838f49e40e3ea757" dependencies = [ "arrow-array", "arrow-buffer", @@ -4117,9 +4173,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -4304,9 +4360,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.2" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2806eaa3524762875e21c3dcd057bc4b7bfa01ce4da8d46be1cd43649e1cc6b" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "oorandom" @@ -4316,9 +4372,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.71" +version = "0.10.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" dependencies = [ "bitflags 2.9.0", "cfg-if", @@ -4348,9 +4404,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.106" +version = "0.9.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" dependencies = [ "cc", "libc", @@ -4523,7 +4579,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.10", + "redox_syscall 0.5.11", "smallvec", "windows-targets 0.52.6", ] @@ -4593,7 +4649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.9.0", ] [[package]] @@ -4797,9 +4853,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", "syn 2.0.100", @@ -4877,9 +4933,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f1c6c3591120564d64db2261bec5f910ae454f01def849b9c22835a84695e86" +checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" dependencies = [ "cfg-if", "indoc", @@ -4896,9 +4952,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9b6c2b34cf71427ea37c7001aefbaeb85886a074795e35f161f5aecc7620a7a" +checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" dependencies = [ "once_cell", "target-lexicon", @@ -4906,9 +4962,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5507651906a46432cdda02cd02dd0319f6064f1374c9147c45b978621d2c3a9c" +checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" dependencies = [ "libc", "pyo3-build-config", @@ -4916,9 +4972,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d394b5b4fd8d97d48336bb0dd2aebabad39f1d294edd6bcd2cccf2eefe6f42" +checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -4928,9 +4984,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd72da09cfa943b1080f621f024d2ef7e2773df7badd51aa30a2be1f8caa7c8e" +checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" dependencies = [ "heck", "proc-macro2", @@ -4941,9 +4997,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.37.3" +version = "0.37.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf763ab1c7a3aa408be466efc86efe35ed1bd3dd74173ed39d6b0d0a6f0ba148" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" dependencies = [ "memchr", "serde", @@ -4961,7 +5017,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.25", + "rustls 0.23.26", "socket2", "thiserror 2.0.12", "tokio", @@ -4980,7 +5036,7 @@ dependencies = [ "rand 0.9.0", "ring", "rustc-hash 2.1.1", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -5135,9 +5191,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ "bitflags 2.9.0", ] @@ -5235,7 +5291,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.8", + "h2 0.4.9", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -5252,7 +5308,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -5331,9 +5387,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.10" +version = "0.10.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652edd001c53df0b3f96a36a8dc93fce6866988efc16808235653c6bcac8bf2" +checksum = "19e8d2cfa184d94d0726d650a9f4a1be7f9b76ac9fdb954219878dc00c1c1e7b" dependencies = [ "bytemuck", "byteorder", @@ -5398,14 +5454,14 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys 0.9.3", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] @@ -5423,9 +5479,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.25" +version = "0.23.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" +checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" dependencies = [ "aws-lc-rs", "log 0.4.27", @@ -5650,9 +5706,9 @@ dependencies = [ [[package]] name = "serde_arrow" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c51448819179b0656880c6b83a7d53c427039cdfcc300dfa0c8a7974c87ce39" +checksum = "a0462b8e06478cd310e8de11ea2e64c214522275a0b537b3879dbed24a9e01b5" dependencies = [ "arrow-array", "arrow-schema", @@ -5698,6 +5754,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.9.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time 0.3.41", +] + +[[package]] +name = "serde_with_macros" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "serial_test" version = "3.2.0" @@ -5824,9 +5910,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "snafu" @@ -5857,9 +5943,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", @@ -6116,7 +6202,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix 1.0.3", + "rustix 1.0.5", "windows-sys 0.59.0", ] @@ -6279,12 +6365,13 @@ dependencies = [ "pgwire", "rand 0.8.5", "regex 1.11.1", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-pemfile 2.2.0", "scopeguard", "serde", "serde_arrow", "serde_json", + "serde_with", "serial_test", "sled", "sqllogictest", @@ -6348,9 +6435,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.1" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", @@ -6427,7 +6514,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.25", + "rustls 0.23.26", "tokio", ] @@ -6467,7 +6554,7 @@ version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ - "indexmap", + "indexmap 2.9.0", "toml_datetime", "winnow", ] @@ -6987,7 +7074,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ - "redox_syscall 0.5.10", + "redox_syscall 0.5.11", "wasite", "web-sys", ] @@ -7025,11 +7112,37 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.52.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" dependencies = [ - "windows-targets 0.52.6", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings 0.4.0", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", ] [[package]] @@ -7045,7 +7158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", - "windows-strings", + "windows-strings 0.3.1", "windows-targets 0.53.0", ] @@ -7067,6 +7180,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -7215,9 +7337,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index cda8431f..7da1ea9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ uuid = { version = "1.13", features = ["v4", "serde"] } serde = { version = "1", features = ["derive"] } serde_arrow = { version = "0.13.1", features = ["arrow-54"] } serde_json = "1.0.138" +serde_with = "3.12" async-trait = "0.1.86" env_logger = "0.11.6" log = "0.4.25" @@ -22,15 +23,14 @@ delta_kernel = { version = "0.8.0", features = [ "arrow-conversion", "default-engine", ] } -chrono = "0.4.39" +chrono = { version = "0.4.39", features = ["serde"] } pgwire = "0.28.0" futures = "0.3.31" bytes = "1.4" tokio-rustls = "0.26.1" sled = "0.34.7" actix-web = "4.9.0" -# datafusion-postgres = { git = "https://github.com/apitoolkit/FusionGate.git" } -datafusion-postgres = { git = "https://github.com/apitoolkit/datafusion-postgres.git", branch = "insert-query-compliance" } +datafusion-postgres = { git = "https://github.com/sunng87/datafusion-postgres.git" } # datafusion-postgres = { path = "../datafusion-projects/datafusion-postgres/datafusion-postgres/" } datafusion-functions-json = "0.46.0" anyhow = "1.0.95" diff --git a/src/database.rs b/src/database.rs index 140764a6..43834dcd 100644 --- a/src/database.rs +++ b/src/database.rs @@ -321,26 +321,19 @@ impl Database { // Checkpoint in the background if needed if should_checkpoint { - // Clone the necessary resources for the background task - let table_ref_clone = Arc::clone(&table_ref); - - // Spawn a background task to perform checkpointing - tokio::spawn(async move { - // Take a read lock for checkpointing - let result = async { - let table = table_ref_clone.read().await; - let version = table.version(); - info!("Starting background checkpointing for Delta table at version {}", version); - checkpoints::create_checkpoint(&table, None).await - }.await; - - match result { - Ok(_) => info!("Background checkpointing completed successfully"), - Err(e) => error!("Background checkpointing failed: {}", e), - } - }); - - info!("Checkpoint scheduled in background"); + // Create a checkpoint immediately within the same function + // This avoids spawning separate tasks that hold onto table references + // which can cause memory leaks + let table = table_ref.read().await; + let version = table.version(); + info!("Starting checkpointing for Delta table at version {}", version); + + match checkpoints::create_checkpoint(&table, None).await { + Ok(_) => info!("Checkpointing completed successfully"), + Err(e) => error!("Checkpointing failed: {}", e), + } + + info!("Checkpoint completed"); } Ok(()) @@ -356,7 +349,7 @@ impl Database { use serde_arrow::schema::SchemaLike; // Convert OtelLogsAndSpans records to Arrow RecordBatch format - let fields = Vec::::from_type::(serde_arrow::schema::TracingOptions::default())?; + let fields = OtelLogsAndSpans::fields()?; let batch = serde_arrow::to_record_batch(&fields, &records)?; // Call insert_records_batch with the converted batch to reuse common insertion logic @@ -618,6 +611,7 @@ mod tests { vec![ OtelLogsAndSpans { project_id: "test_project".to_string(), + // date: timestamp1.date_naive(), timestamp: timestamp1, observed_timestamp: Some(timestamp1), id: "span1".to_string(), @@ -632,6 +626,7 @@ mod tests { }, OtelLogsAndSpans { project_id: "test_project".to_string(), + // date: timestamp2.date_naive(), timestamp: timestamp2, observed_timestamp: Some(timestamp2), id: "span2".to_string(), @@ -860,28 +855,28 @@ mod tests { "| sql_span1a | sql_test_span | 2023-02-01T15:30:00 |", "+------------+---------------+---------------------+", ], &verify_df); - // + let insert_sql = "INSERT INTO otel_logs_and_spans ( - project_id, timestamp, id, - parent_id, name, kind, - status_code, status_message, level, severity___severity_text, severity___severity_number, - body, duration, start_time, end_time - ) VALUES ( - 'test_project', TIMESTAMP '2023-01-01T10:00:00Z', 'sql_span1', - NULL, 'sql_test_span', NULL, - 'OK', 'span inserted successfully', 'INFO', 'INFORMATION', NULL, - NULL, 150000000, TIMESTAMP '2023-01-01T10:00:00Z', NULL - )"; + project_id, date, timestamp, id, hashes, + parent_id, name, kind, + status_code, status_message, level, severity___severity_text, severity___severity_number, + body, duration, start_time, end_time + ) VALUES ( + 'test_project', TIMESTAMP '2023-01-01', TIMESTAMP '2023-01-01T10:00:00Z', 'sql_span1', ARRAY[], + NULL, 'sql_test_span', NULL, + 'OK', 'span inserted successfully', 'INFO', 'INFORMATION', NULL, + NULL, 150000000, TIMESTAMP '2023-01-01T10:00:00Z', NULL + )"; let insert_result = ctx.sql(insert_sql).await?.collect().await?; #[rustfmt::skip] - assert_batches_eq!( - ["+-------+", - "| count |", - "+-------+", - "| 1 |", - "+-------+", - ], &insert_result); + assert_batches_eq!( + ["+-------+", + "| count |", + "+-------+", + "| 1 |", + "+-------+", + ], &insert_result); let verify_df = ctx .sql("SELECT project_id, id, name, timestamp, kind, status_code, severity___severity_text, duration, start_time from otel_logs_and_spans order by timestamp desc") @@ -902,7 +897,7 @@ mod tests { log::info!("Inserting record directly via insert statement"); let insert_sql = "INSERT INTO otel_logs_and_spans ( - project_id, timestamp, observed_timestamp, id, + project_id, date, timestamp, observed_timestamp, id, hashes, parent_id, name, kind, status_code, status_message, level, severity___severity_text, severity___severity_number, body, duration, start_time, end_time, @@ -928,7 +923,7 @@ mod tests { resource___service___version, resource___service___instance___id, resource___service___namespace, resource___telemetry___sdk___language, resource___telemetry___sdk___name, resource___telemetry___sdk___version, resource___user_agent___original ) VALUES ( - 'test_project', TIMESTAMP '2023-01-02T10:00:00Z', NULL, 'sql_span2', + 'test_project','2023-01-02', TIMESTAMP '2023-01-02T10:00:00Z', NULL, 'sql_span2', ARRAY[], NULL, 'sql_test_span', NULL, 'OK', 'span inserted successfully', 'INFO', NULL, NULL, NULL, 150000000, TIMESTAMP '2023-01-01T10:00:00Z', NULL, @@ -957,13 +952,13 @@ mod tests { let insert_result = ctx.sql(insert_sql).await?.collect().await?; #[rustfmt::skip] - assert_batches_eq!( - ["+-------+", - "| count |", - "+-------+", - "| 1 |", - "+-------+", - ], &insert_result); + assert_batches_eq!( + ["+-------+", + "| count |", + "+-------+", + "| 1 |", + "+-------+", + ], &insert_result); // Verify that the SQL-inserted record exists let verify_df = ctx.sql("SELECT id, name, status_message FROM otel_logs_and_spans WHERE id = 'sql_span1'").await?; @@ -1023,23 +1018,23 @@ mod tests { // ], &insert_result); let verify_df = ctx - .sql("SELECT project_id, id, name, timestamp, kind, status_code, severity___severity_text, duration, start_time from otel_logs_and_spans order by timestamp desc") - .await? - .collect() - .await?; + .sql("SELECT project_id, id, name, timestamp, kind, status_code, severity___severity_text, duration, start_time from otel_logs_and_spans order by timestamp desc") + .await? + .collect() + .await?; #[rustfmt::skip] - assert_batches_eq!( - [ - "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", - "| project_id | id | name | timestamp | kind | status_code | severity___severity_text | duration | start_time |", - "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", - "| default | sql_span1a | sql_test_span | 2023-02-01T15:30:00 | | OK | | 150000000 | 2023-02-01T15:30:00 |", - "| test_project | sql_span2 | sql_test_span | 2023-01-02T10:00:00 | | OK | | 150000000 | 2023-01-01T10:00:00 |", - "| test_project | sql_span1 | sql_test_span | 2023-01-01T10:00:00 | | OK | INFORMATION | 150000000 | 2023-01-01T10:00:00 |", - "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", - ], - &verify_df - ); + assert_batches_eq!( + [ + "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", + "| project_id | id | name | timestamp | kind | status_code | severity___severity_text | duration | start_time |", + "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", + "| default | sql_span1a | sql_test_span | 2023-02-01T15:30:00 | | OK | | 150000000 | 2023-02-01T15:30:00 |", + "| test_project | sql_span2 | sql_test_span | 2023-01-02T10:00:00 | | OK | | 150000000 | 2023-01-01T10:00:00 |", + "| test_project | sql_span1 | sql_test_span | 2023-01-01T10:00:00 | | OK | INFORMATION | 150000000 | 2023-01-01T10:00:00 |", + "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", + ], + &verify_df + ); Ok(()) } diff --git a/src/persistent_queue.rs b/src/persistent_queue.rs index 331cdff0..7947adfc 100644 --- a/src/persistent_queue.rs +++ b/src/persistent_queue.rs @@ -1,16 +1,23 @@ +use std::str::FromStr; use std::sync::Arc; -use arrow_schema::{DataType, TimeUnit}; +use arrow_schema::{DataType, FieldRef}; use arrow_schema::{Field, Schema, SchemaRef}; use delta_kernel::schema::StructField; -use serde::{Deserialize, Serialize}; +use log::debug; +use serde::{de::Error as DeError, Deserialize, Deserializer, Serialize}; use serde_arrow::schema::SchemaLike; use serde_arrow::schema::TracingOptions; use serde_json::json; +use serde_with::serde_as; #[allow(non_snake_case)] +#[serde_as] #[derive(Serialize, Deserialize, Clone, Default)] pub struct OtelLogsAndSpans { + #[serde(with = "chrono::serde::ts_microseconds")] + pub timestamp: chrono::DateTime, + #[serde(with = "chrono::serde::ts_microseconds_option")] pub observed_timestamp: Option>, @@ -147,17 +154,27 @@ pub struct OtelLogsAndSpans { // Top-level fields pub project_id: String, - #[serde(with = "chrono::serde::ts_microseconds")] - pub timestamp: chrono::DateTime, + #[serde(default)] + #[serde(deserialize_with = "default_on_empty_string")] + pub date: chrono::NaiveDate, } impl OtelLogsAndSpans { pub fn table_name() -> String { "otel_logs_and_spans".to_string() } - pub fn columns() -> anyhow::Result> { + + pub fn fields() -> anyhow::Result> { let tracing_options = TracingOptions::default() + .strings_as_large_utf8(false) + .sequence_as_large_list(false) + .sequence_as_large_list(false) .overwrite("project_id", json!({"name": "project_id", "data_type": "Utf8", "nullable": false}))? + .overwrite("date", json!({"name": "date", "data_type": "Date32", "nullable": false}))? + .overwrite("duration", json!({"name": "duration", "data_type": "UInt64", "nullable": true}))? + .overwrite("body", json!({"name":"body", "data_type": "Utf8", "nullable": true}))? + .overwrite("attributes", json!({"name":"attributes", "data_type": "Utf8", "nullable": true}))? + .overwrite("resource", json!({"name":"resource", "data_type": "Utf8", "nullable": true}))? .overwrite( "timestamp", json!({"name": "timestamp", "data_type": "Timestamp(Microsecond, None)", "nullable": false}), @@ -176,10 +193,15 @@ impl OtelLogsAndSpans { json!({"name": "end_time", "data_type": "Timestamp(Microsecond, None)", "nullable": true}), )?; - let fields = Vec::::from_type::(tracing_options)?; + Ok(Vec::::from_type::(tracing_options)?) + } + + pub fn columns() -> anyhow::Result> { + let fields = OtelLogsAndSpans::fields()?; let vec_refs: Vec = fields.iter().map(|arc_field| arc_field.as_ref().try_into().unwrap()).collect(); assert_eq!(fields[fields.len() - 2].data_type(), &DataType::Utf8); - assert_eq!(fields[fields.len() - 1].data_type(), &DataType::Timestamp(TimeUnit::Microsecond, None)); + assert_eq!(fields[fields.len() - 1].data_type(), &DataType::Date32); + debug!("schema_field columns {:?}", vec_refs); Ok(vec_refs) } @@ -195,6 +217,21 @@ impl OtelLogsAndSpans { } pub fn partitions() -> Vec { - vec!["project_id".to_string(), "timestamp".to_string()] + vec!["project_id".to_string(), "date".to_string()] + } +} + +pub fn default_on_empty_string<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: Deserialize<'de> + Default + FromStr, + ::Err: std::fmt::Display, +{ + let opt = Option::::deserialize(deserializer)?; + + match opt { + None => Ok(T::default()), + Some(s) if s.is_empty() => Ok(T::default()), + Some(s) => T::from_str(&s).map_err(DeError::custom), } } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index bace96c6..483865d5 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -105,8 +105,9 @@ mod integration { // Insert test data let timestamp_str = format!("'{}'", chrono::Utc::now().format("%Y-%m-%d %H:%M:%S")); let insert_query = format!( - "INSERT INTO otel_logs_and_spans (project_id, timestamp, id, name, status_code, status_message, level) - VALUES ($1, {}, $2, $3, $4, $5, $6)", + "INSERT INTO otel_logs_and_spans (project_id, date, timestamp, id, name, status_code, status_message, level, hashes) + VALUES ($1, {}, {}, $2, $3, $4, $5, $6, ARRAY[])", + chrono::Utc::now().date_naive().to_string(), timestamp_str ); @@ -151,7 +152,7 @@ mod integration { assert_eq!(count_rows[0].get::<_, String>(0), "test_project", "project_id should match"); let count_rows = client.query("SELECT * FROM otel_logs_and_spans WHERE project_id = $1", &[&"test_project"]).await?; - assert_eq!(count_rows[0].columns().len(), 84, "Should return all 84 columns"); + assert_eq!(count_rows[0].columns().len(), 86, "Should return all 84 columns"); Ok::<_, tokio_postgres::Error>(()) } @@ -190,8 +191,9 @@ mod integration { // Create timestamp for the insert query let timestamp_str = format!("'{}'", chrono::Utc::now().format("%Y-%m-%d %H:%M:%S")); let insert_query = format!( - "INSERT INTO otel_logs_and_spans (project_id, timestamp, id, name, status_code, status_message, level) - VALUES ($1, {}, $2, $3, $4, $5, $6)", + "INSERT INTO otel_logs_and_spans (project_id, date, timestamp, id, name, status_code, status_message, level, hashes) + VALUES ($1, {}, {}, $2, $3, $4, $5, $6, ARRAY[])", + chrono::Utc::now().date_naive().to_string(), timestamp_str ); From 78fe58d52d179a05ce86701158a50ee45c298b4e Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Wed, 16 Apr 2025 00:56:47 +0200 Subject: [PATCH 03/33] pin datafusion-postgres version --- Cargo.lock | 2 +- Cargo.toml | 3 ++- src/database.rs | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd140127..df709e0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2443,7 +2443,7 @@ dependencies = [ [[package]] name = "datafusion-postgres" version = "0.3.0" -source = "git+https://github.com/sunng87/datafusion-postgres.git#2cf58787a8bf3e12a82b836d7dbdc5f6aee9f5a6" +source = "git+https://github.com/sunng87/datafusion-postgres.git?rev=2cf58787a8bf3e12a82b836d7dbdc5f6aee9f5a6#2cf58787a8bf3e12a82b836d7dbdc5f6aee9f5a6" dependencies = [ "async-trait", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 7da1ea9d..8d2bda1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,8 @@ bytes = "1.4" tokio-rustls = "0.26.1" sled = "0.34.7" actix-web = "4.9.0" -datafusion-postgres = { git = "https://github.com/sunng87/datafusion-postgres.git" } +datafusion-postgres = { git = "https://github.com/sunng87/datafusion-postgres.git", rev = "2cf58787a8bf3e12a82b836d7dbdc5f6aee9f5a6" } +# datafusion-postgres = { git = "https://github.com/apitoolkit/datafusion-postgres.git", branch = "insert-query-compliance" } # datafusion-postgres = { path = "../datafusion-projects/datafusion-postgres/datafusion-postgres/" } datafusion-functions-json = "0.46.0" anyhow = "1.0.95" diff --git a/src/database.rs b/src/database.rs index 43834dcd..1208fc2d 100644 --- a/src/database.rs +++ b/src/database.rs @@ -8,7 +8,6 @@ use datafusion::common::SchemaExt; use datafusion::execution::context::SessionContext; use datafusion::execution::TaskContext; use datafusion::logical_expr::{Expr, Operator, TableProviderFilterPushDown}; -use datafusion::physical_expr::intervals::utils::check_support; use datafusion::physical_plan::insert::{DataSink, DataSinkExec}; use datafusion::physical_plan::DisplayAs; use datafusion::scalar::ScalarValue; From ca5c0e064db2b2b30a603262cf38c64e93f113bd Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Wed, 16 Apr 2025 01:15:50 +0200 Subject: [PATCH 04/33] example .env --- .env.example | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..f7050a4e --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +AWS_REGION= +AWS_S3_BUCKET= +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +PGWIRE_PORT=5432 +PORT=80 +TIMEFUSION_TABLE_PREFIX=timefusion From 2af695d751e07922d1a8fd11a79b5b7f1e0e9bba Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Thu, 17 Apr 2025 00:05:26 +0200 Subject: [PATCH 05/33] add usage examples --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 44c47554..134a5c15 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,32 @@ Timefusion can be configured using the following environment variables: | AWS_SECRET_ACCESS_KEY | AWS secret key | - | For local development, you can set `QUEUE_DB_PATH` to a location in your development environment. + +## Usage + +There currently exists only 1 table. otel_logs_and_spans. +You can access it via psql: eg if running locally: + +``` +$ psql "postgresql://postgres:postgres@localhost:12345/postgres" + +psql (16.8 (Homebrew), server 0.28.0) +WARNING: psql major version 16, server major version 0.28. + Some psql features might not work. +Type "help" for help. + +postgres=> insert into otel_logs_and_spans (name, id, project_id, hashes, timestamp, date) values ('name3', 'id2', 'pid3', ARRAY[], '2025-04-14 02:00:24.898000', '2025-04-14 02:00:24.898000'); +INSERT 0 1 + +postgres=> select name, id, project_id,timestamp from otel_logs_and_spans limit 10; + name | id | project_id | timestamp +-------------------------------------------------------------+--------------------------------------+--------------------------------------+---------------------------- + GET api/v1/validations/profundity-interior/(?P[^/.]+)/$ | 00000000-09ab-47bc-b628-2554626d1261 | 00000000-876e-41fa-be63-52d5bcfc037e | 2025-04-14 20:45:08.713740 + GET api/v1/validations/tire-pressure/(?P[^/.]+)/$ | 00000000-3d2a-445d-b7bf-3e56125b48d4 | 00000000-876e-41fa-be63-52d5bcfc037e | 2025-04-14 22:01:00.816390 + POST api/v1/validations/warnings-of-wear/$ | 00000000-4ced-48f4-830d-64d3531eb7f0 | 00000000-876e-41fa-be63-52d5bcfc037e | 2025-04-14 21:18:08.635637 + +``` + +``` + +``` From 3243f9e81ae8881a9a8d77592a723837f12f055e Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Sat, 19 Apr 2025 02:09:49 +0200 Subject: [PATCH 06/33] fix schema --- src/persistent_queue.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/persistent_queue.rs b/src/persistent_queue.rs index 7947adfc..8c6d0837 100644 --- a/src/persistent_queue.rs +++ b/src/persistent_queue.rs @@ -110,9 +110,9 @@ pub struct OtelLogsAndSpans { // HTTP https://opentelemetry.io/docs/specs/semconv/http/http-spans/ pub attributes___http___request___method: Option, pub attributes___http___request___method_original: Option, - pub attributes___http___response___status_code: Option, - pub attributes___http___request___resend_count: Option, - pub attributes___http___request___body___size: Option, + pub attributes___http___response___status_code: Option, + pub attributes___http___request___resend_count: Option, + pub attributes___http___request___body___size: Option, // Session https://opentelemetry.io/docs/specs/semconv/general/session/ pub attributes___session___id: Option, From eef2360f3090589891d47c8a5d8500d8e7fd048a Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Sat, 19 Apr 2025 15:37:44 +0200 Subject: [PATCH 07/33] implement a batch queue and make insert query handling async --- .env.example | 10 +++ Cargo.lock | 34 ++++++++ Cargo.toml | 3 + README.md | 21 +++-- src/batch_queue.rs | 157 +++++++++++++++++++++++++++++++++++ src/database.rs | 200 +++++++++++++++++++++++++++++---------------- src/lib.rs | 1 + src/main.rs | 24 ++++-- 8 files changed, 365 insertions(+), 85 deletions(-) create mode 100644 src/batch_queue.rs diff --git a/.env.example b/.env.example index f7050a4e..aa99c8ef 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,13 @@ AWS_SECRET_ACCESS_KEY= PGWIRE_PORT=5432 PORT=80 TIMEFUSION_TABLE_PREFIX=timefusion + +# Batch insert configuration +# Interval between batch inserts in milliseconds (default: 1000) +BATCH_INTERVAL_MS=1000 +# Maximum number of rows to process in a single batch (default: 1000) +MAX_BATCH_SIZE=1000 +# Set to "true" to enable batching queue (default: false = direct insertion) +ENABLE_BATCH_QUEUE=false +# Maximum number of concurrent PostgreSQL connections (default: 100) +MAX_PG_CONNECTIONS=100 diff --git a/Cargo.lock b/Cargo.lock index df709e0f..e028d64b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1833,6 +1833,28 @@ dependencies = [ "time 0.1.45", ] +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1852,6 +1874,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -6347,6 +6378,7 @@ dependencies = [ "chrono", "color-eyre", "criterion", + "crossbeam", "datafusion", "datafusion-common", "datafusion-functions-json", @@ -6376,11 +6408,13 @@ dependencies = [ "sled", "sqllogictest", "sqlparser 0.55.0", + "tap", "task", "tempfile", "tokio", "tokio-postgres", "tokio-rustls 0.26.2", + "tokio-stream", "tokio-util", "tracing", "tracing-opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index 8d2bda1a..d8e9b560 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,9 +40,12 @@ tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } tracing = "0.1.41" dotenv = "0.15.0" task = "0.0.1" +crossbeam = "0.8.4" sqlparser = "0.55.0" rustls-pemfile = "2.2.0" rustls = "0.23.23" +tokio-stream = { version = "0.1.17", features = ["net"] } +tap = "1.0.1" actix-service = "2.0.2" lazy_static = "1.5.0" bcrypt = "0.17.0" diff --git a/README.md b/README.md index 134a5c15..0243870f 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,19 @@ Timefusion speaks the postgres dialect, so you can insert and read from it using Timefusion can be configured using the following environment variables: -| Variable | Description | Default | -| --------------------- | ----------------------------- | -------------------------- | -| `PORT` | HTTP server port | `80` | -| `PGWIRE_PORT` | PostgreSQL wire protocol port | `5432` | -| `AWS_S3_BUCKET` | AWS S3 bucket name | Required | -| `AWS_S3_ENDPOINT` | AWS S3 endpoint URL | `https://s3.amazonaws.com` | -| AWS_ACCESS_KEY_ID | AWS access key | - | -| AWS_SECRET_ACCESS_KEY | AWS secret key | - | +| Variable | Description | Default | +| ---------------------- | ------------------------------------------------ | --------------------------- | +| `PORT` | HTTP server port | `80` | +| `PGWIRE_PORT` | PostgreSQL wire protocol port | `5432` | +| `AWS_S3_BUCKET` | AWS S3 bucket name | Required | +| `AWS_S3_ENDPOINT` | AWS S3 endpoint URL | `https://s3.amazonaws.com` | +| `AWS_ACCESS_KEY_ID` | AWS access key | - | +| `AWS_SECRET_ACCESS_KEY`| AWS secret key | - | +| `TIMEFUSION_TABLE_PREFIX` | Prefix for Delta tables | `timefusion` | +| `BATCH_INTERVAL_MS` | Interval between batch inserts in milliseconds | `1000` | +| `MAX_BATCH_SIZE` | Maximum number of rows in a single batch | `1000` | +| `ENABLE_BATCH_QUEUE` | Whether to use batch queue for inserts | `false` (direct insertion) | +| `MAX_PG_CONNECTIONS` | Maximum number of concurrent PostgreSQL connections | `100` | For local development, you can set `QUEUE_DB_PATH` to a location in your development environment. diff --git a/src/batch_queue.rs b/src/batch_queue.rs new file mode 100644 index 00000000..e9dca9b6 --- /dev/null +++ b/src/batch_queue.rs @@ -0,0 +1,157 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use crossbeam::queue::SegQueue; +use delta_kernel::arrow::record_batch::RecordBatch; +use tokio::sync::RwLock; +use tokio::time::interval; +use tracing::{error, info}; + +/// BatchQueue collects RecordBatches and processes them at intervals +#[derive(Debug)] +pub struct BatchQueue { + queue: Arc>, + is_shutting_down: Arc>, +} + +impl BatchQueue { + pub fn new(db: Arc, interval_ms: u64, max_rows: usize) -> Self { + let queue = Arc::new(SegQueue::new()); + let is_shutting_down = Arc::new(RwLock::new(false)); + + let queue_clone = Arc::clone(&queue); + let shutdown_flag = Arc::clone(&is_shutting_down); + + tokio::spawn(async move { + let mut ticker = interval(Duration::from_millis(interval_ms)); + + loop { + ticker.tick().await; + + if *shutdown_flag.read().await { + process_batches(&db, &queue_clone, max_rows).await; + break; + } + + process_batches(&db, &queue_clone, max_rows).await; + } + }); + + Self { queue, is_shutting_down } + } + + /// Add a batch to the queue + pub fn queue(&self, batch: RecordBatch) -> Result<()> { + if let Ok(flag) = self.is_shutting_down.try_read() { + if *flag { + return Err(anyhow::anyhow!("BatchQueue is shutting down")); + } + } + + self.queue.push(batch); + Ok(()) + } + + /// Signal shutdown and wait for queue to drain + pub async fn shutdown(&self) { + let mut guard = self.is_shutting_down.write().await; + *guard = true; + } +} + +/// Process batches from the queue +async fn process_batches(db: &Arc, queue: &Arc>, max_rows: usize) { + if queue.is_empty() { + return; + } + + let mut batches = Vec::new(); + let mut total_rows = 0; + + // Take batches up to max_rows + while !queue.is_empty() && total_rows < max_rows { + if let Some(batch) = queue.pop() { + total_rows += batch.num_rows(); + batches.push(batch); + } else { + break; + } + } + + if batches.is_empty() { + return; + } + + // Measure and log the insertion performance + let start = Instant::now(); + + // Use skip_queue=true to force direct insertion and avoid infinite loop + match db.insert_records_batch("", batches.clone(), true).await { + Ok(_) => { + let elapsed = start.elapsed(); + info!( + batches_count = batches.len(), + rows_count = total_rows, + duration_ms = elapsed.as_millis(), + "Batch insert completed" + ); + } + Err(e) => { + error!("Failed to insert batches: {}", e); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::database::Database; + use crate::persistent_queue::OtelLogsAndSpans; + use chrono::Utc; + use serde_arrow::schema::SchemaLike; + use std::sync::Arc; + use tokio::time::sleep; + + #[tokio::test] + async fn test_batch_queue() -> Result<()> { + dotenv::dotenv().ok(); + let test_prefix = format!("test-batch-{}", uuid::Uuid::new_v4()); + unsafe { + std::env::set_var("TIMEFUSION_TABLE_PREFIX", &test_prefix); + } + + // Initialize DB + let db = Arc::new(Database::new().await?); + + // Create batch queue with short interval for testing + let batch_queue = BatchQueue::new(Arc::clone(&db), 100, 10); + + // Create test records and convert to RecordBatch + let now = Utc::now(); + let records = (0..5) + .map(|i| OtelLogsAndSpans { + project_id: "default".to_string(), + timestamp: now, + id: format!("test-{}", i), + hashes: vec![], + date: now.date_naive(), + ..Default::default() + }) + .collect::>(); + + let fields = OtelLogsAndSpans::fields()?; + let batch = serde_arrow::to_record_batch(&fields, &records)?; + + // Queue and process the batch + batch_queue.queue(batch)?; + sleep(Duration::from_millis(200)).await; + + // Shutdown queue + batch_queue.shutdown().await; + sleep(Duration::from_millis(200)).await; + + Ok(()) + } +} + diff --git a/src/database.rs b/src/database.rs index 1208fc2d..7d10461e 100644 --- a/src/database.rs +++ b/src/database.rs @@ -18,14 +18,21 @@ use datafusion::{ logical_expr::{dml::InsertOp, BinaryExpr}, physical_plan::{DisplayFormatType, ExecutionPlan, SendableRecordBatchStream}, }; +use datafusion_postgres::{DfSessionService, HandlerFactory}; use delta_kernel::arrow::record_batch::RecordBatch; use deltalake::checkpoints; use deltalake::{storage::StorageOptions, DeltaOps, DeltaTable, DeltaTableBuilder}; use futures::StreamExt; use std::fmt; use std::{any::Any, collections::HashMap, env, sync::Arc}; +use std::{net::SocketAddr, time::Duration}; use tokio::sync::RwLock; +use tokio::{ + net::TcpListener, + time::timeout, +}; use tokio_util::sync::CancellationToken; +use tokio_stream::wrappers::TcpListenerStream; use tracing::{debug, error, info}; use url::Url; @@ -36,12 +43,14 @@ pub type ProjectConfigs = Arc>>; #[derive(Debug)] pub struct Database { project_configs: ProjectConfigs, + batch_queue: Option>, } impl Clone for Database { fn clone(&self) -> Self { Self { project_configs: Arc::clone(&self.project_configs), + batch_queue: self.batch_queue.clone(), } } } @@ -65,6 +74,7 @@ impl Database { let db = Self { project_configs: Arc::new(RwLock::new(project_configs)), + batch_queue: None, // Batch queue is set later }; db.register_project("default", &storage_uri, None, None, None).await?; @@ -72,6 +82,12 @@ impl Database { Ok(db) } + /// Set the batch queue to use for insert operations + pub fn with_batch_queue(mut self, batch_queue: Arc) -> Self { + self.batch_queue = Some(batch_queue); + self + } + /// Create and configure a SessionContext with DataFusion settings pub fn create_session_context(&self) -> SessionContext { use datafusion::config::ConfigOptions; @@ -88,7 +104,12 @@ impl Database { // Create tables and register them with session context let schema = OtelLogsAndSpans::schema_ref(); - let routing_table = ProjectRoutingTable::new("default".to_string(), Arc::new(self.clone()), schema); + + // Get batch queue from the app state if available + let batch_queue = self.batch_queue.as_ref().map(Arc::clone); + + let routing_table = ProjectRoutingTable::new("default".to_string(), Arc::new(self.clone()), schema, batch_queue); + ctx.register_table(OtelLogsAndSpans::table_name(), Arc::new(routing_table))?; info!("Registered ProjectRoutingTable with SessionContext"); @@ -177,79 +198,78 @@ impl Database { ctx.register_udf(set_config_udf); } - /// Start a PGWire server with the given session context pub async fn start_pgwire_server( - &self, session_context: SessionContext, port: u16, shutdown_token: CancellationToken, - ) -> Result> { - use datafusion_postgres::{DfSessionService, HandlerFactory}; - use tokio::net::TcpListener; - - let pg_service = Arc::new(DfSessionService::new(session_context)); - let handler_factory = Arc::new(HandlerFactory(pg_service.clone())); - - info!("Attempting to bind PGWire server to 0.0.0.0:{}", port); - let bind_addr = format!("0.0.0.0:{}", port); - let pg_listener = match TcpListener::bind(&bind_addr).await { - Ok(listener) => { - info!("PGWire server successfully bound to {}", bind_addr); - listener - } - Err(e) => { - error!("Failed to bind PGWire server to {}: {:?}", bind_addr, e); - return Err(anyhow::anyhow!("Failed to bind PGWire server: {:?}", e)); - } - }; + &self, session_ctx: SessionContext, port: u16, shutdown: CancellationToken, + ) -> anyhow::Result> { + // 1) build listener + // Simple binding with clear logging + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + info!("Binding PGWire server to {}...", addr); + + // Use standard tokio TcpListener + let listener = TcpListener::bind(addr).await?; + + // Log successful binding + if let Ok(local_addr) = listener.local_addr() { + info!("PGWire server successfully bound to {}", local_addr); + } - // Log all local addresses this process is listening on - info!("PGWire server running on 0.0.0.0:{}", port); - info!("PGWire server local address: {:?}", pg_listener.local_addr()); + // 2) pgwire service + handler + let service = Arc::new(DfSessionService::new(session_ctx)); + let factory = Arc::new(HandlerFactory(service)); - let pgwire_shutdown = shutdown_token.clone(); + // 3) concurrency + logging + let max_conn = std::env::var("MAX_PG_CONNECTIONS").ok().and_then(|v| v.parse().ok()).unwrap_or(100) as usize; + info!("PGWire listening on 0.0.0.0:{} (limit {})", port, max_conn); - let pg_server = tokio::spawn({ - let handler_factory = handler_factory.clone(); + // 4) spawn the accept‐&‐process loop + let handle = tokio::spawn({ + let shutdown = shutdown.clone(); + let stream = TcpListenerStream::new(listener); async move { - loop { - tokio::select! { - _ = pgwire_shutdown.cancelled() => { - info!("PGWire server shutting down."); - break; - } - result = pg_listener.accept() => { - match result { - Ok((socket, addr)) => { - info!("PGWire: Received connection from {}, preparing to process", addr); - info!("PGWire: Socket details - local addr: {:?}, peer addr: {:?}", - socket.local_addr().map_err(|e| debug!("Failed to get local addr: {:?}", e)), - socket.peer_addr().map_err(|e| debug!("Failed to get peer addr: {:?}", e)) - ); - - let handler_factory = handler_factory.clone(); - tokio::spawn(async move { - info!("PGWire: Started processing connection from {}", addr); - match pgwire::tokio::process_socket(socket, None, handler_factory).await { - Ok(()) => { - info!("PGWire: Connection from {} processed successfully", addr); - } - Err(e) => { - error!("PGWire: Error processing connection from {}: {:?}", addr, e); - error!("PGWire: Error details - {}", e); - } - } - }); + stream + .take_until(shutdown.cancelled()) + .for_each_concurrent(max_conn, |conn| async { + match conn { + Ok(sock) => { + // Set TCP nodelay option for better performance + if let Err(e) = sock.set_nodelay(true) { + error!("Failed to set TCP_NODELAY: {}", e); } - Err(e) => { - error!("PGWire: Error accepting connection: {:?}", e); - error!("PGWire: Connection accept error details - {}", e); + + // Log client connection info + if let Ok(peer_addr) = sock.peer_addr() { + info!("Client connected from {}", peer_addr); + } + + // Use a longer timeout to prevent idle disconnections + let timeout_duration = Duration::from_secs(3600); // 1 hour + info!("Starting PGWire connection processing"); + let start_time = std::time::Instant::now(); + + match timeout(timeout_duration, pgwire::tokio::process_socket(sock, None, factory.clone())).await { + Ok(Ok(_)) => { + let elapsed = start_time.elapsed(); + info!("PGWire connection completed successfully (duration: {:?})", elapsed); + }, + Ok(Err(e)) => { + let elapsed = start_time.elapsed(); + error!("PGWire connection error after {:?}: {}", elapsed, e); + }, + Err(_) => { + error!("PGWire connection timed out after 1 hour"); + }, } } + Err(e) => error!("TCP accept error: {}", e), } - } - } + }) + .await; + info!("PGWire server shut down."); } }); - Ok(pg_server) + Ok(handle) } pub async fn resolve_table(&self, project_id: &str) -> DFResult>> { @@ -298,7 +318,25 @@ impl Database { ))) } - pub async fn insert_records_batch(&self, _table: &str, batch: Vec) -> Result<()> { + pub async fn insert_records_batch(&self, _table: &str, batches: Vec, skip_queue: bool) -> Result<()> { + // Check if we should use the batch queue based on: + // 1. skip_queue parameter (if true, always skip) + // 2. ENABLE_BATCH_QUEUE env var (if set to "true", allow queue usage) + // 3. batch_queue existence + let enable_queue = env::var("ENABLE_BATCH_QUEUE").unwrap_or_else(|_| "false".to_string()) == "true"; + + if !skip_queue && enable_queue && self.batch_queue.is_some() { + let queue = self.batch_queue.as_ref().unwrap(); + // Add to batch queue + for batch in batches { + if let Err(e) = queue.queue(batch) { + return Err(anyhow::anyhow!("Queue error: {}", e)); + } + } + return Ok(()); + } + + // Direct insert logic if skip_queue=true, queue disabled, no batch queue, or when processing from batch queue let (_conn_str, _options, table_ref) = { let configs = self.project_configs.read().await; configs.get("default").ok_or_else(|| anyhow::anyhow!("Project ID '{}' not found", "default"))?.clone() @@ -309,7 +347,7 @@ impl Database { let mut table = table_ref.write().await; // Create the DeltaOps with a clone of the table - let write_op = DeltaOps(table.clone()).write(batch).with_partition_columns(OtelLogsAndSpans::partitions()); + let write_op = DeltaOps(table.clone()).write(batches).with_partition_columns(OtelLogsAndSpans::partitions()); let new_table = write_op.await?; let version = new_table.version(); @@ -352,7 +390,8 @@ impl Database { let batch = serde_arrow::to_record_batch(&fields, &records)?; // Call insert_records_batch with the converted batch to reuse common insertion logic - self.insert_records_batch("default", vec![batch]).await + // In tests we always skip the queue for direct insertion + self.insert_records_batch("default", vec![batch], true).await } pub async fn register_project( @@ -374,7 +413,7 @@ impl Database { storage_options.0.insert("AWS_ALLOW_HTTP".to_string(), "true".to_string()); - let mut table = match DeltaTableBuilder::from_uri(conn_str).with_storage_options(storage_options.0.clone()).with_allow_http(true).load().await { + let table = match DeltaTableBuilder::from_uri(conn_str).with_storage_options(storage_options.0.clone()).with_allow_http(true).load().await { Ok(table) => { // Check if table needs checkpointing - use same threshold as in insert_records_batch let version = table.version(); @@ -411,14 +450,16 @@ pub struct ProjectRoutingTable { default_project: String, database: Arc, schema: SchemaRef, + batch_queue: Option>, } impl ProjectRoutingTable { - pub fn new(default_project: String, database: Arc, schema: SchemaRef) -> Self { + pub fn new(default_project: String, database: Arc, schema: SchemaRef, batch_queue: Option>) -> Self { Self { default_project, database, schema, + batch_queue, } } @@ -501,16 +542,26 @@ impl DataSink for ProjectRoutingTable { } async fn write_all(&self, mut data: SendableRecordBatchStream, _context: &Arc) -> DFResult { - let mut new_batches = vec![]; let mut row_count = 0; + let mut batches = Vec::new(); + + // Collect all batches from the stream while let Some(batch) = data.next().await.transpose()? { row_count += batch.num_rows(); - new_batches.push(batch); + batches.push(batch); } + + if batches.is_empty() { + return Ok(0); + } + + // Let the database handle the queue decision with skip_queue=false + // This means it will use the queue if it's available and not disabled via env var self.database - .insert_records_batch("", new_batches) + .insert_records_batch("", batches, false) .await - .map_err(|e| DataFusionError::Execution(format!("Failed to insert records: {}", e)))?; + .map_err(|e| DataFusionError::Execution(format!("Insert error: {}", e)))?; + Ok(row_count as u64) } @@ -596,7 +647,12 @@ mod tests { datafusion_functions_json::register_all(&mut session_context)?; let schema = OtelLogsAndSpans::schema_ref(); - let routing_table = ProjectRoutingTable::new("default".to_string(), Arc::new(db.clone()), schema); + let routing_table = ProjectRoutingTable::new( + "default".to_string(), + Arc::new(db.clone()), + schema, + None, // No batch queue in tests + ); session_context.register_table(OtelLogsAndSpans::table_name(), Arc::new(routing_table))?; Ok((db, session_context, test_prefix)) diff --git a/src/lib.rs b/src/lib.rs index 149debdb..b4d08dc6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ // lib.rs - Export modules for use in tests +pub mod batch_queue; pub mod database; pub mod persistent_queue; diff --git a/src/main.rs b/src/main.rs index 084108b0..74c7645b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,9 @@ // main.rs +mod batch_queue; mod database; mod persistent_queue; use actix_web::{middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder}; +use batch_queue::BatchQueue; use database::Database; use dotenv::dotenv; use futures::TryFutureExt; @@ -54,15 +56,24 @@ async fn main() -> anyhow::Result<()> { info!("Starting TimeFusion application"); // Initialize database - let db = Database::new().await?; + let mut db = Database::new().await?; info!("Database initialized successfully"); - // Create and setup session context + // Setup batch processing with configurable params + let interval_ms = env::var("BATCH_INTERVAL_MS").ok().and_then(|v| v.parse().ok()).unwrap_or(1000); + let max_size = env::var("MAX_BATCH_SIZE").ok().and_then(|v| v.parse().ok()).unwrap_or(1000); + let enable_queue = env::var("ENABLE_BATCH_QUEUE").unwrap_or_else(|_| "false".to_string()) == "true"; + + // Create batch queue + let batch_queue = Arc::new(BatchQueue::new(Arc::new(db.clone()), interval_ms, max_size)); + info!("Batch queue configured (enabled={}, interval={}ms, max_size={})", enable_queue, interval_ms, max_size); + + // Apply and setup + db = db.with_batch_queue(Arc::clone(&batch_queue)); let session_context = db.create_session_context(); db.setup_session_context(&session_context)?; - info!("Session context setup complete"); - // Wrap database in Arc for sharing + // Wrap for sharing let db = Arc::new(db); let app_info = web::Data::new(AppInfo {}); @@ -132,7 +143,10 @@ async fn main() -> anyhow::Result<()> { _ = pg_server.map_err(|e| error!("PGWire server task failed: {:?}", e)) => {}, _ = http_task.map_err(|e| error!("HTTP server task failed: {:?}", e)) => {}, _ = tokio::signal::ctrl_c() => { - info!("Received Ctrl+C, initiating shutdown."); + info!("Received Ctrl+C, initiating shutdown"); + + // Shutdown in order: batch queue first to flush pending data + batch_queue.shutdown().await; shutdown_token.cancel(); http_server_handle.stop(true).await; sleep(Duration::from_secs(1)).await; From 801152fc659c8eb2f15bc1936be51dd020aa7343 Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Sat, 17 May 2025 13:47:00 -0400 Subject: [PATCH 08/33] checkpoint vacuum --- src/database.rs | 122 +++++++++++++++++++++++++++++++++------- src/persistent_queue.rs | 17 ++++++ 2 files changed, 119 insertions(+), 20 deletions(-) diff --git a/src/database.rs b/src/database.rs index 7d10461e..c4bb3a2a 100644 --- a/src/database.rs +++ b/src/database.rs @@ -21,18 +21,18 @@ use datafusion::{ use datafusion_postgres::{DfSessionService, HandlerFactory}; use delta_kernel::arrow::record_batch::RecordBatch; use deltalake::checkpoints; +use deltalake::datafusion::parquet::basic::{Compression, ZstdLevel}; +use deltalake::datafusion::parquet::file::properties::WriterProperties; +use deltalake::operations::transaction::CommitProperties; use deltalake::{storage::StorageOptions, DeltaOps, DeltaTable, DeltaTableBuilder}; use futures::StreamExt; use std::fmt; use std::{any::Any, collections::HashMap, env, sync::Arc}; use std::{net::SocketAddr, time::Duration}; use tokio::sync::RwLock; -use tokio::{ - net::TcpListener, - time::timeout, -}; -use tokio_util::sync::CancellationToken; +use tokio::{net::TcpListener, time::timeout}; use tokio_stream::wrappers::TcpListenerStream; +use tokio_util::sync::CancellationToken; use tracing::{debug, error, info}; use url::Url; @@ -205,10 +205,10 @@ impl Database { // Simple binding with clear logging let addr = SocketAddr::from(([0, 0, 0, 0], port)); info!("Binding PGWire server to {}...", addr); - + // Use standard tokio TcpListener let listener = TcpListener::bind(addr).await?; - + // Log successful binding if let Ok(local_addr) = listener.local_addr() { info!("PGWire server successfully bound to {}", local_addr); @@ -236,29 +236,29 @@ impl Database { if let Err(e) = sock.set_nodelay(true) { error!("Failed to set TCP_NODELAY: {}", e); } - + // Log client connection info if let Ok(peer_addr) = sock.peer_addr() { info!("Client connected from {}", peer_addr); } - + // Use a longer timeout to prevent idle disconnections let timeout_duration = Duration::from_secs(3600); // 1 hour info!("Starting PGWire connection processing"); let start_time = std::time::Instant::now(); - + match timeout(timeout_duration, pgwire::tokio::process_socket(sock, None, factory.clone())).await { Ok(Ok(_)) => { let elapsed = start_time.elapsed(); info!("PGWire connection completed successfully (duration: {:?})", elapsed); - }, + } Ok(Err(e)) => { let elapsed = start_time.elapsed(); error!("PGWire connection error after {:?}: {}", elapsed, e); - }, + } Err(_) => { error!("PGWire connection timed out after 1 hour"); - }, + } } } Err(e) => error!("TCP accept error: {}", e), @@ -324,7 +324,7 @@ impl Database { // 2. ENABLE_BATCH_QUEUE env var (if set to "true", allow queue usage) // 3. batch_queue existence let enable_queue = env::var("ENABLE_BATCH_QUEUE").unwrap_or_else(|_| "false".to_string()) == "true"; - + if !skip_queue && enable_queue && self.batch_queue.is_some() { let queue = self.batch_queue.as_ref().unwrap(); // Add to batch queue @@ -335,7 +335,7 @@ impl Database { } return Ok(()); } - + // Direct insert logic if skip_queue=true, queue disabled, no batch queue, or when processing from batch queue let (_conn_str, _options, table_ref) = { let configs = self.project_configs.read().await; @@ -346,8 +346,18 @@ impl Database { let should_checkpoint = { let mut table = table_ref.write().await; - // Create the DeltaOps with a clone of the table - let write_op = DeltaOps(table.clone()).write(batches).with_partition_columns(OtelLogsAndSpans::partitions()); + // Create writer properties with ZSTD compression and bloom filters + let writer_properties = WriterProperties::builder() + .set_compression(Compression::ZSTD(ZstdLevel::default())) + .set_bloom_filter_enabled(true) + .set_sorting_columns(Some(OtelLogsAndSpans::sorting_columns())) + .build(); + + // Create the DeltaOps with a clone of the table and ZSTD compression + let write_op = DeltaOps(table.clone()) + .write(batches) + .with_partition_columns(OtelLogsAndSpans::partitions()) + .with_writer_properties(writer_properties); let new_table = write_op.await?; let version = new_table.version(); @@ -356,7 +366,7 @@ impl Database { version > 0 && version % 40 == 0 }; - // Checkpoint in the background if needed + // Checkpoint and vacuum in the background if needed if should_checkpoint { // Create a checkpoint immediately within the same function // This avoids spawning separate tasks that hold onto table references @@ -366,11 +376,25 @@ impl Database { info!("Starting checkpointing for Delta table at version {}", version); match checkpoints::create_checkpoint(&table, None).await { - Ok(_) => info!("Checkpointing completed successfully"), + Ok(_) => { + info!("Checkpointing completed successfully"); + + // Get retention period from environment variable or use default (14 days) + let retention_hours = env::var("TIMEFUSION_VACUUM_RETENTION_HOURS") + .unwrap_or_else(|_| "336".to_string()) // Default: 14 days (336 hours) + .parse::() + .unwrap_or(336); // Default to 14 days if parsing fails + + // Release the read lock + drop(table); + + // Perform vacuum operation to clean up old files after checkpointing + self.vacuum_table(&table_ref, retention_hours).await; + }, Err(e) => error!("Checkpointing failed: {}", e), } - info!("Checkpoint completed"); + info!("Maintenance operations completed"); } Ok(()) @@ -394,6 +418,58 @@ impl Database { self.insert_records_batch("default", vec![batch], true).await } + /// Vacuum the Delta table to clean up old files that are no longer needed + /// This reduces storage costs and improves query performance + async fn vacuum_table(&self, table_ref: &Arc>, retention_hours: u64) { + // Log the start of the vacuum operation + info!("Starting vacuum operation with retention period of {} hours", retention_hours); + + // Get a clone of the table to avoid holding the lock during the operation + let table_clone = { + let table = table_ref.read().await; + table.clone() + }; + + info!("Starting vacuum operation with retention period of {} hours", retention_hours); + + // Perform dry run first to log what would be deleted + match DeltaOps(table_clone.clone()) + .vacuum() + .with_retention_period(chrono::Duration::hours(retention_hours as i64)) + .with_dry_run(true) + .await + { + Ok((_, metrics)) => { + let files_deleted = metrics.files_deleted.len(); + info!("Vacuum dry run identified {} files for deletion", files_deleted); + + if files_deleted > 0 { + // Only proceed with actual vacuum if there are files to delete + match DeltaOps(table_clone) + .vacuum() + .with_retention_period(chrono::Duration::hours(retention_hours as i64)) + .with_enforce_retention_duration(false) // Allow deletion of files newer than default retention + .await + { + Ok((_, metrics)) => { + let actual_files_deleted = metrics.files_deleted.len(); + info!("Vacuum completed successfully, deleted {} files", actual_files_deleted); + // Update the table reference with the vacuumed version + let mut table = table_ref.write().await; + if let Ok(()) = table.update().await { + info!("Table updated after vacuum"); + } else { + error!("Failed to update table after vacuum"); + } + }, + Err(e) => error!("Vacuum operation failed: {}", e), + } + } + }, + Err(e) => error!("Vacuum dry run failed: {}", e), + } + } + pub async fn register_project( &self, project_id: &str, conn_str: &str, access_key: Option<&str>, secret_key: Option<&str>, endpoint: Option<&str>, ) -> Result<()> { @@ -430,11 +506,17 @@ impl Database { // Create the table with project_id partitioning only for now // Timestamp partitioning is likely causing issues with nanosecond precision let delta_ops = DeltaOps::try_from_uri(&conn_str).await?; + let commit_properties = CommitProperties::default().with_create_checkpoint(true).with_cleanup_expired_logs(Some(true)); + + // Create table with ZSTD compression and auto-optimization delta_ops .create() .with_columns(OtelLogsAndSpans::columns().unwrap_or_default()) .with_partition_columns(OtelLogsAndSpans::partitions()) .with_storage_options(storage_options.0.clone()) + .with_commit_properties(commit_properties) + .with_configuration_property(deltalake::TableProperty::AutoOptimizeOptimizeWrite, Some("true")) + .with_configuration_property(deltalake::TableProperty::AutoOptimizeAutoCompact, Some("true")) .await? } }; diff --git a/src/persistent_queue.rs b/src/persistent_queue.rs index 8c6d0837..2a4ef785 100644 --- a/src/persistent_queue.rs +++ b/src/persistent_queue.rs @@ -10,6 +10,7 @@ use serde_arrow::schema::SchemaLike; use serde_arrow::schema::TracingOptions; use serde_json::json; use serde_with::serde_as; +use delta_kernel::parquet::format::SortingColumn; #[allow(non_snake_case)] #[serde_as] @@ -219,6 +220,22 @@ impl OtelLogsAndSpans { pub fn partitions() -> Vec { vec!["project_id".to_string(), "date".to_string()] } + + pub fn sorting_columns() -> Vec { + // Define sorting columns for the parquet files to improve query performance + vec![ + SortingColumn { + column_idx: 0, // timestamp is likely first in the schema + descending: true, // newest first + nulls_first: false, + }, + SortingColumn { + column_idx: 3, // id + descending: false, + nulls_first: false, + } + ] + } } pub fn default_on_empty_string<'de, D, T>(deserializer: D) -> Result From 656ab934d6c3a2e41e1e9e934c9f6119cdea1b6f Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Sat, 17 May 2025 20:39:22 -0400 Subject: [PATCH 09/33] no dry run on vacuum --- src/database.rs | 56 +++++++++++++++-------------------------- src/persistent_queue.rs | 8 ++++++ 2 files changed, 28 insertions(+), 36 deletions(-) diff --git a/src/database.rs b/src/database.rs index c4bb3a2a..0bdf6720 100644 --- a/src/database.rs +++ b/src/database.rs @@ -353,7 +353,7 @@ impl Database { .set_sorting_columns(Some(OtelLogsAndSpans::sorting_columns())) .build(); - // Create the DeltaOps with a clone of the table and ZSTD compression + // Create the DeltaOps with a clone of the table, ZSTD compression, and z-ordering let write_op = DeltaOps(table.clone()) .write(batches) .with_partition_columns(OtelLogsAndSpans::partitions()) @@ -378,19 +378,19 @@ impl Database { match checkpoints::create_checkpoint(&table, None).await { Ok(_) => { info!("Checkpointing completed successfully"); - + // Get retention period from environment variable or use default (14 days) let retention_hours = env::var("TIMEFUSION_VACUUM_RETENTION_HOURS") .unwrap_or_else(|_| "336".to_string()) // Default: 14 days (336 hours) .parse::() .unwrap_or(336); // Default to 14 days if parsing fails - + // Release the read lock drop(table); - + // Perform vacuum operation to clean up old files after checkpointing self.vacuum_table(&table_ref, retention_hours).await; - }, + } Err(e) => error!("Checkpointing failed: {}", e), } @@ -423,50 +423,33 @@ impl Database { async fn vacuum_table(&self, table_ref: &Arc>, retention_hours: u64) { // Log the start of the vacuum operation info!("Starting vacuum operation with retention period of {} hours", retention_hours); - + // Get a clone of the table to avoid holding the lock during the operation let table_clone = { let table = table_ref.read().await; table.clone() }; - - info!("Starting vacuum operation with retention period of {} hours", retention_hours); - - // Perform dry run first to log what would be deleted - match DeltaOps(table_clone.clone()) + + // Directly run vacuum without dry run to delete old files + match DeltaOps(table_clone) .vacuum() .with_retention_period(chrono::Duration::hours(retention_hours as i64)) - .with_dry_run(true) + .with_enforce_retention_duration(false) // Allow deletion of files newer than default retention .await { Ok((_, metrics)) => { let files_deleted = metrics.files_deleted.len(); - info!("Vacuum dry run identified {} files for deletion", files_deleted); + info!("Vacuum completed successfully, deleted {} files", files_deleted); - if files_deleted > 0 { - // Only proceed with actual vacuum if there are files to delete - match DeltaOps(table_clone) - .vacuum() - .with_retention_period(chrono::Duration::hours(retention_hours as i64)) - .with_enforce_retention_duration(false) // Allow deletion of files newer than default retention - .await - { - Ok((_, metrics)) => { - let actual_files_deleted = metrics.files_deleted.len(); - info!("Vacuum completed successfully, deleted {} files", actual_files_deleted); - // Update the table reference with the vacuumed version - let mut table = table_ref.write().await; - if let Ok(()) = table.update().await { - info!("Table updated after vacuum"); - } else { - error!("Failed to update table after vacuum"); - } - }, - Err(e) => error!("Vacuum operation failed: {}", e), - } + // Update the table reference with the vacuumed version + let mut table = table_ref.write().await; + if let Ok(()) = table.update().await { + info!("Table updated after vacuum"); + } else { + error!("Failed to update table after vacuum"); } - }, - Err(e) => error!("Vacuum dry run failed: {}", e), + } + Err(e) => error!("Vacuum operation failed: {}", e), } } @@ -509,6 +492,7 @@ impl Database { let commit_properties = CommitProperties::default().with_create_checkpoint(true).with_cleanup_expired_logs(Some(true)); // Create table with ZSTD compression and auto-optimization + // Note: z-ordering will be applied via sorting_columns in the writer properties delta_ops .create() .with_columns(OtelLogsAndSpans::columns().unwrap_or_default()) diff --git a/src/persistent_queue.rs b/src/persistent_queue.rs index 2a4ef785..8da810a6 100644 --- a/src/persistent_queue.rs +++ b/src/persistent_queue.rs @@ -236,6 +236,14 @@ impl OtelLogsAndSpans { } ] } + + pub fn z_order_columns() -> Vec { + // Define z-order columns for efficient time-series range queries + vec![ + "timestamp".to_string(), + "id".to_string() + ] + } } pub fn default_on_empty_string<'de, D, T>(deserializer: D) -> Result From 50ab97d614294e8a99b4e738d493b448202b5b7f Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Sat, 17 May 2025 22:20:30 -0400 Subject: [PATCH 10/33] implement optimize --- src/database.rs | 65 +++++++++++++++++++++++++++++++++++++++-- src/persistent_queue.rs | 11 +++---- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/src/database.rs b/src/database.rs index 0bdf6720..93e5cc3f 100644 --- a/src/database.rs +++ b/src/database.rs @@ -388,8 +388,15 @@ impl Database { // Release the read lock drop(table); - // Perform vacuum operation to clean up old files after checkpointing - self.vacuum_table(&table_ref, retention_hours).await; + // Optimize the table with z-ordering + match self.optimize_table(&table_ref).await { + Ok(_) => { + info!("Table optimization completed successfully"); + // Perform vacuum operation to clean up old files after optimization + self.vacuum_table(&table_ref, retention_hours).await; + } + Err(e) => error!("Table optimization failed: {}", e), + } } Err(e) => error!("Checkpointing failed: {}", e), } @@ -418,6 +425,58 @@ impl Database { self.insert_records_batch("default", vec![batch], true).await } + /// Optimize the Delta table using Z-ordering on timestamp and id columns + /// This improves query performance for time-based queries + async fn optimize_table(&self, table_ref: &Arc>) -> Result<()> { + // Log the start of the optimization operation + info!("Starting Delta table optimization with Z-ordering"); + + // Get a clone of the table to avoid holding the lock during the operation + let table_clone = { + let table = table_ref.read().await; + table.clone() + }; + + // Run optimize operation with Z-order on the timestamp and id columns + // and a target size of 256MB for optimal file size + let writer_properties = WriterProperties::builder() + .set_compression(Compression::ZSTD(ZstdLevel::default())) + .set_bloom_filter_enabled(true) + .set_sorting_columns(Some(OtelLogsAndSpans::sorting_columns())) + .build(); + + // Note: Z-order functionality is achieved through sorting_columns in writer_properties + let optimize_result = DeltaOps(table_clone) + .optimize() + .with_type(deltalake::operations::optimize::OptimizeType::ZOrder(OtelLogsAndSpans::z_order_columns())) + .with_target_size(268435456) // 256MB + .with_writer_properties(writer_properties) + .await; + + match optimize_result { + Ok((new_table, metrics)) => { + info!( + "Optimization with sorted columns completed: {} files removed, {} files added, {} partitions optimized, {} total files considered, {} files skipped", + metrics.num_files_removed, + metrics.num_files_added, + metrics.partitions_optimized, + metrics.total_considered_files, + metrics.total_files_skipped + ); + + // Update the table reference with the optimized version + let mut table = table_ref.write().await; + *table = new_table; + + Ok(()) + } + Err(e) => { + error!("Optimization operation failed: {}", e); + Err(anyhow::anyhow!("Table optimization failed: {}", e)) + } + } + } + /// Vacuum the Delta table to clean up old files that are no longer needed /// This reduces storage costs and improves query performance async fn vacuum_table(&self, table_ref: &Arc>, retention_hours: u64) { @@ -440,7 +499,7 @@ impl Database { Ok((_, metrics)) => { let files_deleted = metrics.files_deleted.len(); info!("Vacuum completed successfully, deleted {} files", files_deleted); - + // Update the table reference with the vacuumed version let mut table = table_ref.write().await; if let Ok(()) = table.update().await { diff --git a/src/persistent_queue.rs b/src/persistent_queue.rs index 8da810a6..ce554c4f 100644 --- a/src/persistent_queue.rs +++ b/src/persistent_queue.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use arrow_schema::{DataType, FieldRef}; use arrow_schema::{Field, Schema, SchemaRef}; +use delta_kernel::parquet::format::SortingColumn; use delta_kernel::schema::StructField; use log::debug; use serde::{de::Error as DeError, Deserialize, Deserializer, Serialize}; @@ -10,7 +11,6 @@ use serde_arrow::schema::SchemaLike; use serde_arrow::schema::TracingOptions; use serde_json::json; use serde_with::serde_as; -use delta_kernel::parquet::format::SortingColumn; #[allow(non_snake_case)] #[serde_as] @@ -225,7 +225,7 @@ impl OtelLogsAndSpans { // Define sorting columns for the parquet files to improve query performance vec![ SortingColumn { - column_idx: 0, // timestamp is likely first in the schema + column_idx: 0, // timestamp is likely first in the schema descending: true, // newest first nulls_first: false, }, @@ -233,16 +233,13 @@ impl OtelLogsAndSpans { column_idx: 3, // id descending: false, nulls_first: false, - } + }, ] } pub fn z_order_columns() -> Vec { // Define z-order columns for efficient time-series range queries - vec![ - "timestamp".to_string(), - "id".to_string() - ] + vec!["timestamp".to_string()] } } From 4b8f47291749ac37af99630fa8a0b24e2bf13d30 Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Sat, 17 May 2025 23:55:43 -0400 Subject: [PATCH 11/33] schedule vacuum and optimize at an interval --- Cargo.lock | 38 +++++++++++++++ Cargo.toml | 1 + src/database.rs | 123 ++++++++++++++++++++++++++++-------------------- src/main.rs | 2 + 4 files changed, 114 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e028d64b..b9d9c830 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1823,6 +1823,17 @@ dependencies = [ "itertools 0.10.5", ] +[[package]] +name = "cron" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8c3e73077b4b4a6ab1ea5047c37c57aee77657bc8ecd6f29b0af082d0b0c07" +dependencies = [ + "chrono", + "nom", + "once_cell", +] + [[package]] name = "crontab" version = "0.1.0" @@ -4299,6 +4310,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -6412,6 +6434,7 @@ dependencies = [ "task", "tempfile", "tokio", + "tokio-cron-scheduler", "tokio-postgres", "tokio-rustls 0.26.2", "tokio-stream", @@ -6485,6 +6508,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-cron-scheduler" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c2e3a88f827f597799cf70a6f673074e62f3fc5ba5993b2873345c618a29af" +dependencies = [ + "chrono", + "cron", + "num-derive", + "num-traits", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "tokio-macros" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index d8e9b560..18409726 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ aws-types = "1.3.6" aws-sdk-s3 = "1.3.0" url = "2.5.4" datafusion-common = "46.0.0" +tokio-cron-scheduler = "0.10" [dev-dependencies] serial_test = "3.2.0" diff --git a/src/database.rs b/src/database.rs index 93e5cc3f..756d6d16 100644 --- a/src/database.rs +++ b/src/database.rs @@ -44,6 +44,7 @@ pub type ProjectConfigs = Arc>>; pub struct Database { project_configs: ProjectConfigs, batch_queue: Option>, + maintenance_shutdown: Arc, } impl Clone for Database { @@ -51,6 +52,7 @@ impl Clone for Database { Self { project_configs: Arc::clone(&self.project_configs), batch_queue: self.batch_queue.clone(), + maintenance_shutdown: Arc::clone(&self.maintenance_shutdown), } } } @@ -75,6 +77,7 @@ impl Database { let db = Self { project_configs: Arc::new(RwLock::new(project_configs)), batch_queue: None, // Batch queue is set later + maintenance_shutdown: Arc::new(CancellationToken::new()), }; db.register_project("default", &storage_uri, None, None, None).await?; @@ -87,6 +90,65 @@ impl Database { self.batch_queue = Some(batch_queue); self } + + /// Start background maintenance schedulers for optimize and vacuum operations + pub async fn start_maintenance_schedulers(self) -> Result { + use tokio_cron_scheduler::{Job, JobScheduler}; + + let scheduler = JobScheduler::new().await?; + let db = Arc::new(self.clone()); + + // Optimize job - every 3 hours + let optimize_job = Job::new_async("0 0 */3 * * *", { + let db = db.clone(); + move |_, _| { + let db = db.clone(); + Box::pin(async move { + info!("Running optimize on all tables"); + for (project_id, (_, _, table)) in db.project_configs.read().await.iter() { + if let Err(e) = db.optimize_table(table).await { + error!("Optimize failed for {}: {}", project_id, e); + } + } + }) + } + })?; + + scheduler.add(optimize_job).await?; + + // Vacuum job - daily at 3AM + let vacuum_job = Job::new_async("0 0 3 * * *", { + let db = db.clone(); + move |_, _| { + let db = db.clone(); + Box::pin(async move { + info!("Running vacuum on all tables"); + let retention_hours = env::var("TIMEFUSION_VACUUM_RETENTION_HOURS") + .unwrap_or_else(|_| "336".to_string()) + .parse::().unwrap_or(336); + + for (project_id, (_, _, table)) in db.project_configs.read().await.iter() { + info!("Vacuuming {} (retention: {}h)", project_id, retention_hours); + db.vacuum_table(table, retention_hours).await; + } + }) + } + })?; + + scheduler.add(vacuum_job).await?; + + // Start the scheduler + scheduler.start().await?; + + // Handle shutdown + let shutdown = self.maintenance_shutdown.clone(); + tokio::spawn(async move { + shutdown.cancelled().await; + info!("Shutting down maintenance scheduler"); + }); + + Ok(self) + } /// Create and configure a SessionContext with DataFusion settings pub fn create_session_context(&self) -> SessionContext { @@ -342,66 +404,27 @@ impl Database { configs.get("default").ok_or_else(|| anyhow::anyhow!("Project ID '{}' not found", "default"))?.clone() }; + // Create writer properties with ZSTD compression and bloom filters + let writer_properties = WriterProperties::builder() + .set_compression(Compression::ZSTD(ZstdLevel::default())) + .set_bloom_filter_enabled(true) + .set_sorting_columns(Some(OtelLogsAndSpans::sorting_columns())) + .build(); + // Scope the write lock to minimize lock time - let should_checkpoint = { + { let mut table = table_ref.write().await; - // Create writer properties with ZSTD compression and bloom filters - let writer_properties = WriterProperties::builder() - .set_compression(Compression::ZSTD(ZstdLevel::default())) - .set_bloom_filter_enabled(true) - .set_sorting_columns(Some(OtelLogsAndSpans::sorting_columns())) - .build(); - - // Create the DeltaOps with a clone of the table, ZSTD compression, and z-ordering + // Create the DeltaOps with a clone of the table let write_op = DeltaOps(table.clone()) .write(batches) .with_partition_columns(OtelLogsAndSpans::partitions()) .with_writer_properties(writer_properties); let new_table = write_op.await?; - let version = new_table.version(); *table = new_table; - - version > 0 && version % 40 == 0 - }; - - // Checkpoint and vacuum in the background if needed - if should_checkpoint { - // Create a checkpoint immediately within the same function - // This avoids spawning separate tasks that hold onto table references - // which can cause memory leaks - let table = table_ref.read().await; - let version = table.version(); - info!("Starting checkpointing for Delta table at version {}", version); - - match checkpoints::create_checkpoint(&table, None).await { - Ok(_) => { - info!("Checkpointing completed successfully"); - - // Get retention period from environment variable or use default (14 days) - let retention_hours = env::var("TIMEFUSION_VACUUM_RETENTION_HOURS") - .unwrap_or_else(|_| "336".to_string()) // Default: 14 days (336 hours) - .parse::() - .unwrap_or(336); // Default to 14 days if parsing fails - - // Release the read lock - drop(table); - - // Optimize the table with z-ordering - match self.optimize_table(&table_ref).await { - Ok(_) => { - info!("Table optimization completed successfully"); - // Perform vacuum operation to clean up old files after optimization - self.vacuum_table(&table_ref, retention_hours).await; - } - Err(e) => error!("Table optimization failed: {}", e), - } - } - Err(e) => error!("Checkpointing failed: {}", e), - } - - info!("Maintenance operations completed"); + + // Note: Checkpointing, optimization, and vacuum are now managed by scheduled jobs } Ok(()) diff --git a/src/main.rs b/src/main.rs index 74c7645b..2f3f3936 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,6 +70,8 @@ async fn main() -> anyhow::Result<()> { // Apply and setup db = db.with_batch_queue(Arc::clone(&batch_queue)); + // Start maintenance schedulers for regular optimize and vacuum + db = db.start_maintenance_schedulers().await?; let session_context = db.create_session_context(); db.setup_session_context(&session_context)?; From 1a8796c6a60b582183b661766b345f4a53a8f370 Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Sun, 18 May 2025 01:21:33 -0400 Subject: [PATCH 12/33] compact and optimize immediately --- src/database.rs | 58 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/src/database.rs b/src/database.rs index 756d6d16..0ee9d69d 100644 --- a/src/database.rs +++ b/src/database.rs @@ -90,21 +90,47 @@ impl Database { self.batch_queue = Some(batch_queue); self } - + /// Start background maintenance schedulers for optimize and vacuum operations pub async fn start_maintenance_schedulers(self) -> Result { use tokio_cron_scheduler::{Job, JobScheduler}; - + let scheduler = JobScheduler::new().await?; let db = Arc::new(self.clone()); - + + // Run immediate optimize and vacuum operations at startup + info!("Running immediate optimize and vacuum on startup"); + let startup_db = db.clone(); + tokio::spawn(async move { + info!("Starting immediate optimize operation on all tables"); + + // Run optimize first + for (project_id, (_, _, table)) in startup_db.project_configs.read().await.iter() { + info!("Optimizing table for project '{}' on startup", project_id); + if let Err(e) = startup_db.optimize_table(table).await { + error!("Startup optimize failed for {}: {}", project_id, e); + } + } + + // Then run vacuum on the optimized tables + let retention_hours = env::var("TIMEFUSION_VACUUM_RETENTION_HOURS").unwrap_or_else(|_| "336".to_string()).parse::().unwrap_or(336); + info!("Starting immediate vacuum operation on all tables (retention: {}h)", retention_hours); + + for (project_id, (_, _, table)) in startup_db.project_configs.read().await.iter() { + info!("Vacuuming table for project '{}' on startup", project_id); + startup_db.vacuum_table(table, retention_hours).await; + } + + info!("Completed startup maintenance operations"); + }); + // Optimize job - every 3 hours let optimize_job = Job::new_async("0 0 */3 * * *", { let db = db.clone(); move |_, _| { let db = db.clone(); Box::pin(async move { - info!("Running optimize on all tables"); + info!("Running scheduled optimize on all tables"); for (project_id, (_, _, table)) in db.project_configs.read().await.iter() { if let Err(e) = db.optimize_table(table).await { error!("Optimize failed for {}: {}", project_id, e); @@ -113,20 +139,18 @@ impl Database { }) } })?; - + scheduler.add(optimize_job).await?; - + // Vacuum job - daily at 3AM let vacuum_job = Job::new_async("0 0 3 * * *", { let db = db.clone(); move |_, _| { let db = db.clone(); Box::pin(async move { - info!("Running vacuum on all tables"); - let retention_hours = env::var("TIMEFUSION_VACUUM_RETENTION_HOURS") - .unwrap_or_else(|_| "336".to_string()) - .parse::().unwrap_or(336); - + info!("Running scheduled vacuum on all tables"); + let retention_hours = env::var("TIMEFUSION_VACUUM_RETENTION_HOURS").unwrap_or_else(|_| "336".to_string()).parse::().unwrap_or(336); + for (project_id, (_, _, table)) in db.project_configs.read().await.iter() { info!("Vacuuming {} (retention: {}h)", project_id, retention_hours); db.vacuum_table(table, retention_hours).await; @@ -134,19 +158,19 @@ impl Database { }) } })?; - + scheduler.add(vacuum_job).await?; - + // Start the scheduler scheduler.start().await?; - + // Handle shutdown let shutdown = self.maintenance_shutdown.clone(); tokio::spawn(async move { shutdown.cancelled().await; info!("Shutting down maintenance scheduler"); }); - + Ok(self) } @@ -423,7 +447,7 @@ impl Database { let new_table = write_op.await?; *table = new_table; - + // Note: Checkpointing, optimization, and vacuum are now managed by scheduled jobs } @@ -736,7 +760,7 @@ impl TableProvider for ProjectRoutingTable { // Create a physical plan from the logical plan. // Check that the schema of the plan matches the schema of this table. match self.schema().logically_equivalent_names_and_types(&input.schema()) { - Ok(_) => info!("Schema validation passed"), + Ok(_) => debug!("insert_into; Schema validation passed"), Err(e) => { error!("Schema validation failed: {}", e); return Err(e); From 9065dde653bd4ab948ac44181670dff09ad57a38 Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Sun, 18 May 2025 02:19:41 -0400 Subject: [PATCH 13/33] optimize and vacuum --- src/database.rs | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/database.rs b/src/database.rs index 0ee9d69d..78d505c2 100644 --- a/src/database.rs +++ b/src/database.rs @@ -98,32 +98,6 @@ impl Database { let scheduler = JobScheduler::new().await?; let db = Arc::new(self.clone()); - // Run immediate optimize and vacuum operations at startup - info!("Running immediate optimize and vacuum on startup"); - let startup_db = db.clone(); - tokio::spawn(async move { - info!("Starting immediate optimize operation on all tables"); - - // Run optimize first - for (project_id, (_, _, table)) in startup_db.project_configs.read().await.iter() { - info!("Optimizing table for project '{}' on startup", project_id); - if let Err(e) = startup_db.optimize_table(table).await { - error!("Startup optimize failed for {}: {}", project_id, e); - } - } - - // Then run vacuum on the optimized tables - let retention_hours = env::var("TIMEFUSION_VACUUM_RETENTION_HOURS").unwrap_or_else(|_| "336".to_string()).parse::().unwrap_or(336); - info!("Starting immediate vacuum operation on all tables (retention: {}h)", retention_hours); - - for (project_id, (_, _, table)) in startup_db.project_configs.read().await.iter() { - info!("Vacuuming table for project '{}' on startup", project_id); - startup_db.vacuum_table(table, retention_hours).await; - } - - info!("Completed startup maintenance operations"); - }); - // Optimize job - every 3 hours let optimize_job = Job::new_async("0 0 */3 * * *", { let db = db.clone(); From 905bfc3b0c96b2b32904d74ac8b24adce4b2d002 Mon Sep 17 00:00:00 2001 From: Anthony Alaribe Date: Wed, 21 May 2025 10:19:42 -0700 Subject: [PATCH 14/33] zstd level 6 --- src/batch_queue.rs | 1 - src/database.rs | 18 +++++++++--------- src/main.rs | 9 ++++++--- src/persistent_queue.rs | 2 +- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/batch_queue.rs b/src/batch_queue.rs index e9dca9b6..074eac99 100644 --- a/src/batch_queue.rs +++ b/src/batch_queue.rs @@ -154,4 +154,3 @@ mod tests { Ok(()) } } - diff --git a/src/database.rs b/src/database.rs index 78d505c2..6b1bc132 100644 --- a/src/database.rs +++ b/src/database.rs @@ -3,19 +3,19 @@ use anyhow::Result; use arrow_schema::SchemaRef; use async_trait::async_trait; use datafusion::arrow::array::Array; -use datafusion::common::not_impl_err; use datafusion::common::SchemaExt; -use datafusion::execution::context::SessionContext; +use datafusion::common::not_impl_err; use datafusion::execution::TaskContext; +use datafusion::execution::context::SessionContext; use datafusion::logical_expr::{Expr, Operator, TableProviderFilterPushDown}; -use datafusion::physical_plan::insert::{DataSink, DataSinkExec}; use datafusion::physical_plan::DisplayAs; +use datafusion::physical_plan::insert::{DataSink, DataSinkExec}; use datafusion::scalar::ScalarValue; use datafusion::{ catalog::Session, datasource::{TableProvider, TableType}, error::{DataFusionError, Result as DFResult}, - logical_expr::{dml::InsertOp, BinaryExpr}, + logical_expr::{BinaryExpr, dml::InsertOp}, physical_plan::{DisplayFormatType, ExecutionPlan, SendableRecordBatchStream}, }; use datafusion_postgres::{DfSessionService, HandlerFactory}; @@ -24,7 +24,7 @@ use deltalake::checkpoints; use deltalake::datafusion::parquet::basic::{Compression, ZstdLevel}; use deltalake::datafusion::parquet::file::properties::WriterProperties; use deltalake::operations::transaction::CommitProperties; -use deltalake::{storage::StorageOptions, DeltaOps, DeltaTable, DeltaTableBuilder}; +use deltalake::{DeltaOps, DeltaTable, DeltaTableBuilder, storage::StorageOptions}; use futures::StreamExt; use std::fmt; use std::{any::Any, collections::HashMap, env, sync::Arc}; @@ -228,7 +228,7 @@ impl Database { pub fn register_set_config_udf(&self, ctx: &SessionContext) { use datafusion::arrow::array::{StringArray, StringBuilder}; use datafusion::arrow::datatypes::DataType; - use datafusion::logical_expr::{create_udf, ColumnarValue, ScalarFunctionImplementation, Volatility}; + use datafusion::logical_expr::{ColumnarValue, ScalarFunctionImplementation, Volatility, create_udf}; let set_config_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| -> datafusion::error::Result { let param_value_array = match &args[1] { @@ -402,9 +402,9 @@ impl Database { configs.get("default").ok_or_else(|| anyhow::anyhow!("Project ID '{}' not found", "default"))?.clone() }; - // Create writer properties with ZSTD compression and bloom filters + // Create writer properties with ZSTD compression level 6 and bloom filters let writer_properties = WriterProperties::builder() - .set_compression(Compression::ZSTD(ZstdLevel::default())) + .set_compression(Compression::ZSTD(ZstdLevel::try_new(6).unwrap())) .set_bloom_filter_enabled(true) .set_sorting_columns(Some(OtelLogsAndSpans::sorting_columns())) .build(); @@ -461,7 +461,7 @@ impl Database { // Run optimize operation with Z-order on the timestamp and id columns // and a target size of 256MB for optimal file size let writer_properties = WriterProperties::builder() - .set_compression(Compression::ZSTD(ZstdLevel::default())) + .set_compression(Compression::ZSTD(ZstdLevel::try_new(6).unwrap())) .set_bloom_filter_enabled(true) .set_sorting_columns(Some(OtelLogsAndSpans::sorting_columns())) .build(); diff --git a/src/main.rs b/src/main.rs index 2f3f3936..b04f384f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,14 +2,14 @@ mod batch_queue; mod database; mod persistent_queue; -use actix_web::{middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder}; +use actix_web::{App, HttpResponse, HttpServer, Responder, middleware::Logger, post, web}; use batch_queue::BatchQueue; use database::Database; use dotenv::dotenv; use futures::TryFutureExt; use serde::Deserialize; use std::{env, sync::Arc}; -use tokio::time::{sleep, Duration}; +use tokio::time::{Duration, sleep}; use tokio_util::sync::CancellationToken; use tracing::{error, info}; use tracing_subscriber::EnvFilter; @@ -66,7 +66,10 @@ async fn main() -> anyhow::Result<()> { // Create batch queue let batch_queue = Arc::new(BatchQueue::new(Arc::new(db.clone()), interval_ms, max_size)); - info!("Batch queue configured (enabled={}, interval={}ms, max_size={})", enable_queue, interval_ms, max_size); + info!( + "Batch queue configured (enabled={}, interval={}ms, max_size={})", + enable_queue, interval_ms, max_size + ); // Apply and setup db = db.with_batch_queue(Arc::clone(&batch_queue)); diff --git a/src/persistent_queue.rs b/src/persistent_queue.rs index ce554c4f..0987b3a9 100644 --- a/src/persistent_queue.rs +++ b/src/persistent_queue.rs @@ -6,7 +6,7 @@ use arrow_schema::{Field, Schema, SchemaRef}; use delta_kernel::parquet::format::SortingColumn; use delta_kernel::schema::StructField; use log::debug; -use serde::{de::Error as DeError, Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, de::Error as DeError}; use serde_arrow::schema::SchemaLike; use serde_arrow::schema::TracingOptions; use serde_json::json; From 2c830a0405fa68ae5343082f8620d4d9f631b934 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 30 May 2025 23:37:32 +0100 Subject: [PATCH 15/33] feat : object store foundation laid --- Cargo.lock | 536 ++++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 + benches/benchmarks.rs | 176 ++++++------- src/batch_queue.rs | 20 +- src/database.rs | 87 ++++--- src/lib.rs | 1 + src/main.rs | 7 +- src/object_store_cache.rs | 99 +++++++ src/persistent_queue.rs | 132 +++++----- tests/integration_test.rs | 9 +- tests/sqllogictest.rs | 11 +- 11 files changed, 845 insertions(+), 235 deletions(-) create mode 100644 src/object_store_cache.rs diff --git a/Cargo.lock b/Cargo.lock index b9d9c830..d01efa5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -370,12 +370,27 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "array-init" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" +[[package]] +name = "array-util" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e509844de8f09b90a2c3444684a2b6695f4071360e13d2fda0af9f749cc2ed6" +dependencies = [ + "arrayvec", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -601,6 +616,18 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.4.19" @@ -618,6 +645,34 @@ dependencies = [ "zstd-safe", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.88" @@ -644,6 +699,18 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "auto_enums" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c170965892137a3a9aeb000b4524aa3cc022a310e709d848b6e1cdce4ab4781" +dependencies = [ + "derive_utils", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -1587,7 +1654,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] @@ -1617,6 +1684,15 @@ dependencies = [ "cc", ] +[[package]] +name = "cmsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c840ee51da812c6cd621f9f7e07dfb00a49f91283a8e6380c78cba4f61aba" +dependencies = [ + "paste", +] + [[package]] name = "color-eyre" version = "0.6.3" @@ -1660,6 +1736,15 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1959,14 +2044,38 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] @@ -1979,17 +2088,28 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.11.1", "syn 2.0.100", ] +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", "quote", "syn 2.0.100", ] @@ -2045,7 +2165,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log 0.4.27", - "object_store", + "object_store 0.11.2", "parking_lot 0.12.3", "parquet", "rand 0.8.5", @@ -2097,7 +2217,7 @@ dependencies = [ "datafusion-physical-plan", "futures", "log 0.4.27", - "object_store", + "object_store 0.11.2", "tokio", ] @@ -2116,7 +2236,7 @@ dependencies = [ "indexmap 2.9.0", "libc", "log 0.4.27", - "object_store", + "object_store 0.11.2", "parquet", "paste", "recursive", @@ -2160,7 +2280,7 @@ dependencies = [ "glob", "itertools 0.14.0", "log 0.4.27", - "object_store", + "object_store 0.11.2", "rand 0.8.5", "tokio", "tokio-util", @@ -2187,7 +2307,7 @@ dependencies = [ "datafusion-expr", "futures", "log 0.4.27", - "object_store", + "object_store 0.11.2", "parking_lot 0.12.3", "rand 0.8.5", "tempfile", @@ -2506,7 +2626,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-proto-common", - "object_store", + "object_store 0.11.2", "prost", ] @@ -2564,7 +2684,7 @@ dependencies = [ "home", "indexmap 2.9.0", "itertools 0.13.0", - "object_store", + "object_store 0.11.2", "parquet", "reqwest", "roaring", @@ -2620,7 +2740,7 @@ dependencies = [ "deltalake-core", "futures", "maplit", - "object_store", + "object_store 0.11.2", "regex 1.11.1", "thiserror 2.0.12", "tokio", @@ -2673,7 +2793,7 @@ dependencies = [ "num-bigint", "num-traits", "num_cpus", - "object_store", + "object_store 0.11.2", "parking_lot 0.12.3", "parquet", "percent-encoding", @@ -2759,6 +2879,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "derive_utils" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "digest" version = "0.10.7" @@ -2787,6 +2918,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dunce" version = "1.0.5" @@ -2917,6 +3054,27 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "eyre" version = "0.6.12" @@ -2995,6 +3153,18 @@ dependencies = [ "miniz_oxide 0.8.8", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3031,6 +3201,113 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "635c7077026867cb5e5ea576c461f29b1c4151fce7a9d7cc3a1b1a9902d95c65" +dependencies = [ + "anyhow", + "equivalent", + "foyer-common", + "foyer-memory", + "foyer-storage", + "madsim-tokio", + "mixtrics", + "pin-project", + "serde", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-common" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ed2316785e80137c7b91bb74dab1dc1967c3272df05825397b73ae8fc527041" +dependencies = [ + "ahash 0.8.11", + "bytes", + "cfg-if", + "itertools 0.14.0", + "madsim-tokio", + "mixtrics", + "parking_lot 0.12.3", + "pin-project", + "thiserror 2.0.12", + "tokio", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090cf5b89d49fd61e7da9bfae3a1aef605f03196d542b2f8171c74f3add013f4" +dependencies = [ + "ahash 0.8.11", + "arc-swap", + "bitflags 2.9.0", + "cmsketch", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "hashbrown 0.15.2", + "itertools 0.14.0", + "madsim-tokio", + "mixtrics", + "parking_lot 0.12.3", + "pin-project", + "serde", + "thiserror 2.0.12", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-storage" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "095e857c97d6339d4a4a6424b88d08fe08ad0366bfbfaf65d6ddf55baf3d2a38" +dependencies = [ + "ahash 0.8.11", + "allocator-api2", + "anyhow", + "array-util", + "auto_enums", + "bytes", + "clap", + "equivalent", + "flume", + "foyer-common", + "foyer-memory", + "fs4", + "futures-core", + "futures-util", + "itertools 0.14.0", + "libc", + "lz4", + "madsim-tokio", + "ordered_hash_map", + "parking_lot 0.12.3", + "paste", + "pin-project", + "rand 0.9.0", + "thiserror 2.0.12", + "tokio", + "tracing", + "twox-hash 2.1.0", + "zstd", +] + [[package]] name = "fs-err" version = "3.1.0" @@ -3050,6 +3327,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix 1.0.5", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -3279,6 +3566,15 @@ dependencies = [ "ahash 0.7.8", ] +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash 0.8.11", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -4101,13 +4397,32 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lz4_flex" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" dependencies = [ - "twox-hash", + "twox-hash 1.6.3", ] [[package]] @@ -4121,6 +4436,60 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "madsim" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3c97f34bb19cf6a435a4da2187e90acc6bc59faa730e493b28b6d33e1bb9ccb" +dependencies = [ + "ahash 0.8.11", + "async-channel", + "async-stream", + "async-task", + "bincode", + "bytes", + "downcast-rs", + "futures-util", + "lazy_static", + "libc", + "madsim-macros", + "naive-timer", + "panic-message", + "rand 0.8.5", + "rand_xoshiro", + "rustversion", + "serde", + "spin", + "tokio", + "tokio-util", + "toml", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "madsim-macros" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "madsim-tokio" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3eb2acc57c82d21d699119b859e2df70a91dbdb84734885a1e72be83bdecb5" +dependencies = [ + "madsim", + "spin", + "tokio", +] + [[package]] name = "maplit" version = "1.0.2" @@ -4234,6 +4603,31 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mixtrics" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "749ed12bab176c8a42c13a679dd2de12876d5ad4abe7525548e31ae001a9ebbf" +dependencies = [ + "itertools 0.14.0", + "parking_lot 0.12.3", +] + +[[package]] +name = "naive-timer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" + +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.15", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -4411,6 +4805,30 @@ dependencies = [ "walkdir", ] +[[package]] +name = "object_store" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d94ac16b433c0ccf75326388c893d2835ab7457ea35ab8ba5d745c053ef5fa16" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "http 1.3.1", + "humantime", + "itertools 0.14.0", + "parking_lot 0.12.3", + "percent-encoding", + "thiserror 2.0.12", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -4554,6 +4972,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered_hash_map" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0e5f22bf6dd04abd854a8874247813a8fa2c8c1260eba6fbb150270ce7c176" +dependencies = [ + "hashbrown 0.13.2", +] + [[package]] name = "outref" version = "0.5.2" @@ -4589,6 +5016,18 @@ dependencies = [ "sha2", ] +[[package]] +name = "panic-message" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.11.2" @@ -4662,14 +5101,14 @@ dependencies = [ "lz4_flex", "num", "num-bigint", - "object_store", + "object_store 0.11.2", "paste", "seq-macro", "simdutf8", "snap", "thrift", "tokio", - "twox-hash", + "twox-hash 1.6.3", "zstd", "zstd-sys", ] @@ -5193,6 +5632,15 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rayon" version = "1.10.0" @@ -5795,6 +6243,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5831,7 +6288,7 @@ version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", "syn 2.0.100", @@ -6004,6 +6461,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.6.0" @@ -6115,6 +6581,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -6410,9 +6882,11 @@ dependencies = [ "deltalake", "dotenv", "env_logger", + "foyer", "futures", "lazy_static", "log 0.4.27", + "object_store 0.12.1", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", @@ -6614,11 +7088,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -6627,6 +7116,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ "indexmap 2.9.0", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] @@ -6785,6 +7276,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "twox-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "typenum" version = "1.18.0" diff --git a/Cargo.toml b/Cargo.toml index 18409726..ac48c971 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,8 @@ edition = "2024" [dependencies] tokio = { version = "1.43", features = ["full"] } +foyer = "0.17.0" +object_store = "0.12.1" datafusion = "46.0.0" arrow = "54.2.0" uuid = { version = "1.13", features = ["v4", "serde"] } diff --git a/benches/benchmarks.rs b/benches/benchmarks.rs index a03c6641..17441e23 100644 --- a/benches/benchmarks.rs +++ b/benches/benchmarks.rs @@ -34,51 +34,51 @@ fn bench_batch_ingestion(c: &mut Criterion) { let mut records = Vec::with_capacity(batch_size); for _ in 0..batch_size { records.push(IngestRecord { - table_name: "bench_table".to_string(), - project_id: "bench_project".to_string(), - id: Uuid::new_v4().to_string(), - version: 1, - event_type: "bench_event".to_string(), - timestamp: "2025-03-11T12:00:00Z".to_string(), - trace_id: "trace".to_string(), - span_id: "span".to_string(), - parent_span_id: None, - trace_state: None, - start_time: "2025-03-11T12:00:00Z".to_string(), - end_time: Some("2025-03-11T12:00:01Z".to_string()), - duration_ns: 1_000_000_000, - span_name: "span_name".to_string(), - span_kind: "client".to_string(), - span_type: "bench".to_string(), - status: None, - status_code: 0, - status_message: "OK".to_string(), - severity_text: None, - severity_number: 0, - host: "localhost".to_string(), - url_path: "/".to_string(), - raw_url: "/".to_string(), - method: "GET".to_string(), - referer: "".to_string(), - path_params: None, - query_params: None, - request_headers: None, - response_headers: None, - request_body: None, - response_body: None, - endpoint_hash: "hash".to_string(), - shape_hash: "shape".to_string(), - format_hashes: vec!["fmt".to_string()], - field_hashes: vec!["field".to_string()], - sdk_type: "rust".to_string(), - service_version: None, - attributes: None, - events: None, - links: None, - resource: None, + table_name: "bench_table".to_string(), + project_id: "bench_project".to_string(), + id: Uuid::new_v4().to_string(), + version: 1, + event_type: "bench_event".to_string(), + timestamp: "2025-03-11T12:00:00Z".to_string(), + trace_id: "trace".to_string(), + span_id: "span".to_string(), + parent_span_id: None, + trace_state: None, + start_time: "2025-03-11T12:00:00Z".to_string(), + end_time: Some("2025-03-11T12:00:01Z".to_string()), + duration_ns: 1_000_000_000, + span_name: "span_name".to_string(), + span_kind: "client".to_string(), + span_type: "bench".to_string(), + status: None, + status_code: 0, + status_message: "OK".to_string(), + severity_text: None, + severity_number: 0, + host: "localhost".to_string(), + url_path: "/".to_string(), + raw_url: "/".to_string(), + method: "GET".to_string(), + referer: "".to_string(), + path_params: None, + query_params: None, + request_headers: None, + response_headers: None, + request_body: None, + response_body: None, + endpoint_hash: "hash".to_string(), + shape_hash: "shape".to_string(), + format_hashes: vec!["fmt".to_string()], + field_hashes: vec!["field".to_string()], + sdk_type: "rust".to_string(), + service_version: None, + attributes: None, + events: None, + links: None, + resource: None, instrumentation_scope: None, - errors: None, - tags: vec!["tag".to_string()], + errors: None, + tags: vec!["tag".to_string()], }); } @@ -110,51 +110,51 @@ fn bench_insertion_range(c: &mut Criterion) { let mut records = Vec::with_capacity(size); for _ in 0..size { records.push(IngestRecord { - table_name: "bench_table".to_string(), - project_id: "bench_project".to_string(), - id: Uuid::new_v4().to_string(), - version: 1, - event_type: "bench_event".to_string(), - timestamp: "2025-03-11T12:00:00Z".to_string(), - trace_id: "trace".to_string(), - span_id: "span".to_string(), - parent_span_id: None, - trace_state: None, - start_time: "2025-03-11T12:00:00Z".to_string(), - end_time: Some("2025-03-11T12:00:01Z".to_string()), - duration_ns: 1_000_000_000, - span_name: "span_name".to_string(), - span_kind: "client".to_string(), - span_type: "bench".to_string(), - status: None, - status_code: 0, - status_message: "OK".to_string(), - severity_text: None, - severity_number: 0, - host: "localhost".to_string(), - url_path: "/".to_string(), - raw_url: "/".to_string(), - method: "GET".to_string(), - referer: "".to_string(), - path_params: None, - query_params: None, - request_headers: None, - response_headers: None, - request_body: None, - response_body: None, - endpoint_hash: "hash".to_string(), - shape_hash: "shape".to_string(), - format_hashes: vec!["fmt".to_string()], - field_hashes: vec!["field".to_string()], - sdk_type: "rust".to_string(), - service_version: None, - attributes: None, - events: None, - links: None, - resource: None, + table_name: "bench_table".to_string(), + project_id: "bench_project".to_string(), + id: Uuid::new_v4().to_string(), + version: 1, + event_type: "bench_event".to_string(), + timestamp: "2025-03-11T12:00:00Z".to_string(), + trace_id: "trace".to_string(), + span_id: "span".to_string(), + parent_span_id: None, + trace_state: None, + start_time: "2025-03-11T12:00:00Z".to_string(), + end_time: Some("2025-03-11T12:00:01Z".to_string()), + duration_ns: 1_000_000_000, + span_name: "span_name".to_string(), + span_kind: "client".to_string(), + span_type: "bench".to_string(), + status: None, + status_code: 0, + status_message: "OK".to_string(), + severity_text: None, + severity_number: 0, + host: "localhost".to_string(), + url_path: "/".to_string(), + raw_url: "/".to_string(), + method: "GET".to_string(), + referer: "".to_string(), + path_params: None, + query_params: None, + request_headers: None, + response_headers: None, + request_body: None, + response_body: None, + endpoint_hash: "hash".to_string(), + shape_hash: "shape".to_string(), + format_hashes: vec!["fmt".to_string()], + field_hashes: vec!["field".to_string()], + sdk_type: "rust".to_string(), + service_version: None, + attributes: None, + events: None, + links: None, + resource: None, instrumentation_scope: None, - errors: None, - tags: vec!["tag".to_string()], + errors: None, + tags: vec!["tag".to_string()], }); } diff --git a/src/batch_queue.rs b/src/batch_queue.rs index 074eac99..6e7a53e1 100644 --- a/src/batch_queue.rs +++ b/src/batch_queue.rs @@ -1,17 +1,18 @@ -use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; use anyhow::Result; use crossbeam::queue::SegQueue; use delta_kernel::arrow::record_batch::RecordBatch; -use tokio::sync::RwLock; -use tokio::time::interval; +use tokio::{sync::RwLock, time::interval}; use tracing::{error, info}; /// BatchQueue collects RecordBatches and processes them at intervals #[derive(Debug)] pub struct BatchQueue { - queue: Arc>, + queue: Arc>, is_shutting_down: Arc>, } @@ -105,14 +106,15 @@ async fn process_batches(db: &Arc, queue: &Arc Result<()> { dotenv::dotenv().ok(); diff --git a/src/database.rs b/src/database.rs index 6b1bc132..1d239911 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,57 +1,58 @@ -use crate::persistent_queue::OtelLogsAndSpans; +use std::{any::Any, collections::HashMap, env, fmt, net::SocketAddr, sync::Arc, time::Duration}; + use anyhow::Result; use arrow_schema::SchemaRef; use async_trait::async_trait; -use datafusion::arrow::array::Array; -use datafusion::common::SchemaExt; -use datafusion::common::not_impl_err; -use datafusion::execution::TaskContext; -use datafusion::execution::context::SessionContext; -use datafusion::logical_expr::{Expr, Operator, TableProviderFilterPushDown}; -use datafusion::physical_plan::DisplayAs; -use datafusion::physical_plan::insert::{DataSink, DataSinkExec}; -use datafusion::scalar::ScalarValue; use datafusion::{ + arrow::array::Array, catalog::Session, + common::{SchemaExt, not_impl_err}, datasource::{TableProvider, TableType}, error::{DataFusionError, Result as DFResult}, - logical_expr::{BinaryExpr, dml::InsertOp}, - physical_plan::{DisplayFormatType, ExecutionPlan, SendableRecordBatchStream}, + execution::{TaskContext, context::SessionContext}, + logical_expr::{BinaryExpr, Expr, Operator, TableProviderFilterPushDown, dml::InsertOp}, + physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, SendableRecordBatchStream, + insert::{DataSink, DataSinkExec}, + }, + scalar::ScalarValue, }; use datafusion_postgres::{DfSessionService, HandlerFactory}; use delta_kernel::arrow::record_batch::RecordBatch; -use deltalake::checkpoints; -use deltalake::datafusion::parquet::basic::{Compression, ZstdLevel}; -use deltalake::datafusion::parquet::file::properties::WriterProperties; -use deltalake::operations::transaction::CommitProperties; -use deltalake::{DeltaOps, DeltaTable, DeltaTableBuilder, storage::StorageOptions}; +use deltalake::{ + DeltaOps, DeltaTable, DeltaTableBuilder, checkpoints, + datafusion::parquet::{ + basic::{Compression, ZstdLevel}, + file::properties::WriterProperties, + }, + operations::transaction::CommitProperties, + storage::StorageOptions, +}; use futures::StreamExt; -use std::fmt; -use std::{any::Any, collections::HashMap, env, sync::Arc}; -use std::{net::SocketAddr, time::Duration}; -use tokio::sync::RwLock; -use tokio::{net::TcpListener, time::timeout}; +use tokio::{net::TcpListener, sync::RwLock, time::timeout}; use tokio_stream::wrappers::TcpListenerStream; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info}; use url::Url; +use crate::persistent_queue::OtelLogsAndSpans; + type ProjectConfig = (String, StorageOptions, Arc>); pub type ProjectConfigs = Arc>>; #[derive(Debug)] pub struct Database { - project_configs: ProjectConfigs, - batch_queue: Option>, + project_configs: ProjectConfigs, + batch_queue: Option>, maintenance_shutdown: Arc, } impl Clone for Database { fn clone(&self) -> Self { Self { - project_configs: Arc::clone(&self.project_configs), - batch_queue: self.batch_queue.clone(), + project_configs: Arc::clone(&self.project_configs), + batch_queue: self.batch_queue.clone(), maintenance_shutdown: Arc::clone(&self.maintenance_shutdown), } } @@ -75,8 +76,8 @@ impl Database { let project_configs = HashMap::new(); let db = Self { - project_configs: Arc::new(RwLock::new(project_configs)), - batch_queue: None, // Batch queue is set later + project_configs: Arc::new(RwLock::new(project_configs)), + batch_queue: None, // Batch queue is set later maintenance_shutdown: Arc::new(CancellationToken::new()), }; @@ -150,8 +151,7 @@ impl Database { /// Create and configure a SessionContext with DataFusion settings pub fn create_session_context(&self) -> SessionContext { - use datafusion::config::ConfigOptions; - use datafusion::execution::context::SessionContext; + use datafusion::{config::ConfigOptions, execution::context::SessionContext}; let mut options = ConfigOptions::new(); let _ = options.set("datafusion.sql_parser.enable_information_schema", "true"); @@ -181,9 +181,11 @@ impl Database { /// Register PostgreSQL settings table for compatibility pub fn register_pg_settings_table(&self, ctx: &SessionContext) -> datafusion::error::Result<()> { - use datafusion::arrow::array::StringArray; - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::arrow::record_batch::RecordBatch; + use datafusion::arrow::{ + array::StringArray, + datatypes::{DataType, Field, Schema}, + record_batch::RecordBatch, + }; let schema = Arc::new(Schema::new(vec![ Field::new("name", DataType::Utf8, false), @@ -226,9 +228,13 @@ impl Database { /// Register set_config UDF for PostgreSQL compatibility pub fn register_set_config_udf(&self, ctx: &SessionContext) { - use datafusion::arrow::array::{StringArray, StringBuilder}; - use datafusion::arrow::datatypes::DataType; - use datafusion::logical_expr::{ColumnarValue, ScalarFunctionImplementation, Volatility, create_udf}; + use datafusion::{ + arrow::{ + array::{StringArray, StringBuilder}, + datatypes::DataType, + }, + logical_expr::{ColumnarValue, ScalarFunctionImplementation, Volatility, create_udf}, + }; let set_config_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| -> datafusion::error::Result { let param_value_array = match &args[1] { @@ -594,9 +600,9 @@ impl Database { #[derive(Debug, Clone)] pub struct ProjectRoutingTable { default_project: String, - database: Arc, - schema: SchemaRef, - batch_queue: Option>, + database: Arc, + schema: SchemaRef, + batch_queue: Option>, } impl ProjectRoutingTable { @@ -769,8 +775,7 @@ impl TableProvider for ProjectRoutingTable { #[cfg(test)] mod tests { use chrono::{TimeZone, Utc}; - use datafusion::assert_batches_eq; - use datafusion::prelude::SessionContext; + use datafusion::{assert_batches_eq, prelude::SessionContext}; use dotenv::dotenv; use serial_test::serial; use uuid::Uuid; diff --git a/src/lib.rs b/src/lib.rs index b4d08dc6..460f8eed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ // lib.rs - Export modules for use in tests pub mod batch_queue; pub mod database; +pub mod object_store_cache; pub mod persistent_queue; diff --git a/src/main.rs b/src/main.rs index b04f384f..3971aa1f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,13 +2,14 @@ mod batch_queue; mod database; mod persistent_queue; +use std::{env, sync::Arc}; + use actix_web::{App, HttpResponse, HttpServer, Responder, middleware::Logger, post, web}; use batch_queue::BatchQueue; use database::Database; use dotenv::dotenv; use futures::TryFutureExt; use serde::Deserialize; -use std::{env, sync::Arc}; use tokio::time::{Duration, sleep}; use tokio_util::sync::CancellationToken; use tracing::{error, info}; @@ -20,10 +21,10 @@ struct AppInfo {} #[derive(Deserialize)] struct RegisterProjectRequest { project_id: String, - bucket: String, + bucket: String, access_key: String, secret_key: String, - endpoint: Option, + endpoint: Option, } #[post("/register_project")] diff --git a/src/object_store_cache.rs b/src/object_store_cache.rs new file mode 100644 index 00000000..a866c345 --- /dev/null +++ b/src/object_store_cache.rs @@ -0,0 +1,99 @@ +use std::{path::PathBuf, sync::Arc}; + +use anyhow::Result; +use bytes::Bytes; +use foyer::{Cache, CacheBuilder}; +use object_store::{ObjectStore, path::Path}; +use tokio::sync::RwLock; + +/// A hybrid cache implementation for object store using foyer +pub struct ObjectStoreCache { + cache: Arc>>, + object_store: Arc, +} + +impl ObjectStoreCache { + /// Create a new ObjectStoreCache instance (in-memory cache) + pub fn new( + object_store: Arc, + _cache_dir: PathBuf, // unused for in-memory + capacity: usize, + ) -> Result { + let cache = CacheBuilder::new(capacity).build(); + Ok(Self { + cache: Arc::new(RwLock::new(cache)), + object_store, + }) + } + + /// Get an object from the cache or object store + pub async fn get(&self, path: &Path) -> Result { + // Try to get from cache first + if let Some(entry) = self.cache.read().await.get(path) { + return Ok(entry.value().clone()); + } + + // If not in cache, get from object store + let bytes = self.object_store.get(path).await?.bytes().await?; + + // Store in cache + self.cache.write().await.insert(path.clone(), bytes.clone()); + + Ok(bytes) + } + + /// Put an object into both cache and object store + pub async fn put(&self, path: &Path, bytes: Bytes) -> Result<()> { + // Store in object store + self.object_store.put(path, bytes.clone().into()).await?; + + // Store in cache + self.cache.write().await.insert(path.clone(), bytes); + + Ok(()) + } + + /// Remove an object from both cache and object store + pub async fn remove(&self, path: &Path) -> Result<()> { + // Remove from object store + self.object_store.delete(path).await?; + + // Remove from cache + self.cache.write().await.remove(path); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use object_store::memory::InMemory; + use tempfile::tempdir; + + use super::*; + + #[tokio::test] + async fn test_object_store_cache() -> Result<()> { + let temp_dir = tempdir()?; + let object_store = Arc::new(InMemory::new()); + let cache = ObjectStoreCache::new( + object_store, + temp_dir.path().to_path_buf(), + 1024 * 1024, // 1MB cache + )?; + + let path = Path::from("test.txt"); + let data = Bytes::from("test data"); + + // Test put and get + cache.put(&path, data.clone()).await?; + let retrieved = cache.get(&path).await?; + assert_eq!(retrieved, data); + + // Test remove + cache.remove(&path).await?; + assert!(cache.get(&path).await.is_err()); + + Ok(()) + } +} diff --git a/src/persistent_queue.rs b/src/persistent_queue.rs index 0987b3a9..c8d00466 100644 --- a/src/persistent_queue.rs +++ b/src/persistent_queue.rs @@ -1,14 +1,10 @@ -use std::str::FromStr; -use std::sync::Arc; +use std::{str::FromStr, sync::Arc}; -use arrow_schema::{DataType, FieldRef}; -use arrow_schema::{Field, Schema, SchemaRef}; -use delta_kernel::parquet::format::SortingColumn; -use delta_kernel::schema::StructField; +use arrow_schema::{DataType, Field, FieldRef, Schema, SchemaRef}; +use delta_kernel::{parquet::format::SortingColumn, schema::StructField}; use log::debug; use serde::{Deserialize, Deserializer, Serialize, de::Error as DeError}; -use serde_arrow::schema::SchemaLike; -use serde_arrow::schema::TracingOptions; +use serde_arrow::schema::{SchemaLike, TracingOptions}; use serde_json::json; use serde_with::serde_as; @@ -22,12 +18,12 @@ pub struct OtelLogsAndSpans { #[serde(with = "chrono::serde::ts_microseconds_option")] pub observed_timestamp: Option>, - pub id: String, - pub parent_id: Option, - pub hashes: Vec, // all relevant hashes can be stored here for item identification - pub name: Option, - pub kind: Option, // logs, span, request - pub status_code: Option, + pub id: String, + pub parent_id: Option, + pub hashes: Vec, // all relevant hashes can be stored here for item identification + pub name: Option, + pub kind: Option, // logs, span, request + pub status_code: Option, pub status_message: Option, // Logs specific @@ -36,7 +32,7 @@ pub struct OtelLogsAndSpans { // Severity pub severity: Option, // severity as json - pub severity___severity_text: Option, + pub severity___severity_text: Option, pub severity___severity_number: Option, pub body: Option, // body as json json @@ -46,16 +42,16 @@ pub struct OtelLogsAndSpans { #[serde(with = "chrono::serde::ts_microseconds_option")] pub start_time: Option>, #[serde(with = "chrono::serde::ts_microseconds_option")] - pub end_time: Option>, + pub end_time: Option>, // Context - pub context: Option, // context as json + pub context: Option, // context as json // - pub context___trace_id: Option, - pub context___span_id: Option, + pub context___trace_id: Option, + pub context___span_id: Option, pub context___trace_state: Option, pub context___trace_flags: Option, - pub context___is_remote: Option, + pub context___is_remote: Option, // Events pub events: Option, // events json @@ -64,96 +60,96 @@ pub struct OtelLogsAndSpans { pub links: Option, // links json // Attributes - pub attributes: Option, // attirbutes object as json + pub attributes: Option, // attirbutes object as json // Server and client pub attributes___client___address: Option, - pub attributes___client___port: Option, + pub attributes___client___port: Option, pub attributes___server___address: Option, - pub attributes___server___port: Option, + pub attributes___server___port: Option, // network https://opentelemetry.io/docs/specs/semconv/attributes-registry/network/ - pub attributes___network___local__address: Option, - pub attributes___network___local__port: Option, - pub attributes___network___peer___address: Option, - pub attributes___network___peer__port: Option, - pub attributes___network___protocol___name: Option, + pub attributes___network___local__address: Option, + pub attributes___network___local__port: Option, + pub attributes___network___peer___address: Option, + pub attributes___network___peer__port: Option, + pub attributes___network___protocol___name: Option, pub attributes___network___protocol___version: Option, - pub attributes___network___transport: Option, - pub attributes___network___type: Option, + pub attributes___network___transport: Option, + pub attributes___network___type: Option, // Source Code Attributes - pub attributes___code___number: Option, - pub attributes___code___file___path: Option, + pub attributes___code___number: Option, + pub attributes___code___file___path: Option, pub attributes___code___function___name: Option, - pub attributes___code___line___number: Option, - pub attributes___code___stacktrace: Option, + pub attributes___code___line___number: Option, + pub attributes___code___stacktrace: Option, // Log records. https://opentelemetry.io/docs/specs/semconv/general/logs/ pub attributes___log__record___original: Option, - pub attributes___log__record___uid: Option, + pub attributes___log__record___uid: Option, // Exception https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/ - pub attributes___error___type: Option, - pub attributes___exception___type: Option, - pub attributes___exception___message: Option, + pub attributes___error___type: Option, + pub attributes___exception___type: Option, + pub attributes___exception___message: Option, pub attributes___exception___stacktrace: Option, // URL https://opentelemetry.io/docs/specs/semconv/attributes-registry/url/ pub attributes___url___fragment: Option, - pub attributes___url___full: Option, - pub attributes___url___path: Option, - pub attributes___url___query: Option, - pub attributes___url___scheme: Option, + pub attributes___url___full: Option, + pub attributes___url___path: Option, + pub attributes___url___query: Option, + pub attributes___url___scheme: Option, // Useragent https://opentelemetry.io/docs/specs/semconv/attributes-registry/user-agent/ pub attributes___user_agent___original: Option, // HTTP https://opentelemetry.io/docs/specs/semconv/http/http-spans/ - pub attributes___http___request___method: Option, + pub attributes___http___request___method: Option, pub attributes___http___request___method_original: Option, - pub attributes___http___response___status_code: Option, - pub attributes___http___request___resend_count: Option, - pub attributes___http___request___body___size: Option, + pub attributes___http___response___status_code: Option, + pub attributes___http___request___resend_count: Option, + pub attributes___http___request___body___size: Option, // Session https://opentelemetry.io/docs/specs/semconv/general/session/ - pub attributes___session___id: Option, + pub attributes___session___id: Option, pub attributes___session___previous___id: Option, // Database https://opentelemetry.io/docs/specs/semconv/database/database-spans/ - pub attributes___db___system___name: Option, - pub attributes___db___collection___name: Option, - pub attributes___db___namespace: Option, - pub attributes___db___operation___name: Option, - pub attributes___db___response___status_code: Option, + pub attributes___db___system___name: Option, + pub attributes___db___collection___name: Option, + pub attributes___db___namespace: Option, + pub attributes___db___operation___name: Option, + pub attributes___db___response___status_code: Option, pub attributes___db___operation___batch___size: Option, - pub attributes___db___query___summary: Option, - pub attributes___db___query___text: Option, + pub attributes___db___query___summary: Option, + pub attributes___db___query___text: Option, // https://opentelemetry.io/docs/specs/semconv/attributes-registry/user/ - pub attributes___user___id: Option, - pub attributes___user___email: Option, + pub attributes___user___id: Option, + pub attributes___user___email: Option, pub attributes___user___full_name: Option, - pub attributes___user___name: Option, - pub attributes___user___hash: Option, + pub attributes___user___name: Option, + pub attributes___user___hash: Option, // Resource pub resource: Option, // resource as json // Resource Attributes (subset) https://opentelemetry.io/docs/specs/semconv/resource/ - pub resource___service___name: Option, - pub resource___service___version: Option, + pub resource___service___name: Option, + pub resource___service___version: Option, pub resource___service___instance___id: Option, - pub resource___service___namespace: Option, + pub resource___service___namespace: Option, pub resource___telemetry___sdk___language: Option, - pub resource___telemetry___sdk___name: Option, - pub resource___telemetry___sdk___version: Option, + pub resource___telemetry___sdk___name: Option, + pub resource___telemetry___sdk___version: Option, pub resource___user_agent___original: Option, // Kept at the bottom to make delta-rs happy, so its schema matches datafusion. // Seems delta removes the partition ids from the normal schema and moves them to the end. // Top-level fields - pub project_id: String, + pub project_id: String, #[serde(default)] #[serde(deserialize_with = "default_on_empty_string")] @@ -225,13 +221,13 @@ impl OtelLogsAndSpans { // Define sorting columns for the parquet files to improve query performance vec![ SortingColumn { - column_idx: 0, // timestamp is likely first in the schema - descending: true, // newest first + column_idx: 0, // timestamp is likely first in the schema + descending: true, // newest first nulls_first: false, }, SortingColumn { - column_idx: 3, // id - descending: false, + column_idx: 3, // id + descending: false, nulls_first: false, }, ] diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 483865d5..01d55516 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,13 +1,16 @@ #[cfg(test)] mod integration { + use std::{ + collections::HashSet, + sync::{Arc, Mutex}, + time::{Duration, Instant}, + }; + use anyhow::Result; use dotenv::dotenv; use rand::Rng; use scopeguard; use serial_test::serial; - use std::collections::HashSet; - use std::sync::{Arc, Mutex}; - use std::time::{Duration, Instant}; use timefusion::database::Database; use tokio::{sync::Notify, time::sleep}; use tokio_postgres::{Client, NoTls}; diff --git a/tests/sqllogictest.rs b/tests/sqllogictest.rs index e7832e1f..ebade9d4 100644 --- a/tests/sqllogictest.rs +++ b/tests/sqllogictest.rs @@ -1,15 +1,16 @@ #[cfg(test)] mod sqllogictest_tests { - use anyhow::Result; - use async_trait::async_trait; - use dotenv::dotenv; - use serial_test::serial; - use sqllogictest::{AsyncDB, DBOutput, DefaultColumnType}; use std::{ path::Path, sync::Arc, time::{Duration, Instant}, }; + + use anyhow::Result; + use async_trait::async_trait; + use dotenv::dotenv; + use serial_test::serial; + use sqllogictest::{AsyncDB, DBOutput, DefaultColumnType}; use timefusion::database::Database; use tokio::{sync::Notify, time::sleep}; use tokio_postgres::{NoTls, Row}; From 30bea44b5f9fb8ce721cbf4f4a209688b948df09 Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 1 Jun 2025 23:26:49 +0100 Subject: [PATCH 16/33] a file based implementation --- src/lib.rs | 1 + src/object_store_cache.rs | 225 +++++++++++++++++++++++----- src/object_store_cache_in_memory.rs | 95 ++++++++++++ 3 files changed, 281 insertions(+), 40 deletions(-) create mode 100644 src/object_store_cache_in_memory.rs diff --git a/src/lib.rs b/src/lib.rs index 460f8eed..58a41bb5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,4 +2,5 @@ pub mod batch_queue; pub mod database; pub mod object_store_cache; +pub mod object_store_cache_in_memory; pub mod persistent_queue; diff --git a/src/object_store_cache.rs b/src/object_store_cache.rs index a866c345..eccbbf79 100644 --- a/src/object_store_cache.rs +++ b/src/object_store_cache.rs @@ -1,77 +1,219 @@ use std::{path::PathBuf, sync::Arc}; +use std::fmt::{Debug, Formatter}; +use std::ops::Range; +use std::time::{Duration, Instant}; +use std::fs::remove_dir_all; +use std::io; use anyhow::Result; -use bytes::Bytes; +use bytes::{Bytes, BytesMut}; use foyer::{Cache, CacheBuilder}; -use object_store::{ObjectStore, path::Path}; +use object_store::{ObjectStore, path::Path, PutPayload}; use tokio::sync::RwLock; +use tracing::{debug, error, warn}; +use metrics::{counter, describe_counter, describe_histogram, gauge, histogram, Counter, Gauge}; + +/// Constants for cache configuration +pub const DEFAULT_MIN_FETCH_SIZE: u64 = 1024 * 1024; // 1 MiB +pub const DEFAULT_CACHE_CAPACITY: u64 = 1024 * 1024 * 1024; // 1 GiB +pub const DEFAULT_CACHE_ENTRY_TTL: Duration = Duration::from_secs(3 * 60); + +/// Cache key that includes both path and range information +#[derive(Clone, Hash, Eq, PartialEq)] +pub struct CacheKey { + path: Path, + range: Range, +} + +impl Debug for CacheKey { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}-{:?}", self.path, self.range) + } +} + +impl CacheKey { + fn as_filename(&self) -> String { + format!( + "{}-{}-{}", + self.path.to_string().replace('/', "_"), + self.range.start, + self.range.end + ) + } +} + +/// Metrics for the object store cache +#[derive(Clone)] +pub struct ObjectStoreCacheMetrics { + cache_hits: Counter, + cache_misses: Counter, + cache_evictions: Counter, + cache_size: Gauge, + cache_capacity: Gauge, + cache_read_latency: Counter, + cache_write_latency: Counter, +} + +impl ObjectStoreCacheMetrics { + pub fn new() -> Self { + describe_counter!("object_store_cache_hits", "Number of cache hits"); + describe_counter!("object_store_cache_misses", "Number of cache misses"); + describe_counter!("object_store_cache_evictions", "Number of cache evictions"); + describe_gauge!("object_store_cache_size", "Current cache size in bytes"); + describe_gauge!("object_store_cache_capacity", "Cache capacity in bytes"); + describe_histogram!("object_store_cache_read_latency", "Cache read latency in seconds"); + describe_histogram!("object_store_cache_write_latency", "Cache write latency in seconds"); + + Self { + cache_hits: counter!("object_store_cache_hits"), + cache_misses: counter!("object_store_cache_misses"), + cache_evictions: counter!("object_store_cache_evictions"), + cache_size: gauge!("object_store_cache_size"), + cache_capacity: gauge!("object_store_cache_capacity"), + cache_read_latency: counter!("object_store_cache_read_latency"), + cache_write_latency: counter!("object_store_cache_write_latency"), + } + } +} /// A hybrid cache implementation for object store using foyer pub struct ObjectStoreCache { - cache: Arc>>, + cache: Arc>>, object_store: Arc, + min_fetch_size: u64, + max_cache_size: u64, + base_path: PathBuf, + metrics: ObjectStoreCacheMetrics, } impl ObjectStoreCache { - /// Create a new ObjectStoreCache instance (in-memory cache) + /// Create a new ObjectStoreCache instance pub fn new( object_store: Arc, - _cache_dir: PathBuf, // unused for in-memory - capacity: usize, - ) -> Result { - let cache = CacheBuilder::new(capacity).build(); - Ok(Self { + base_path: PathBuf, + min_fetch_size: u64, + max_cache_size: u64, + ttl: Duration, + ) -> Self { + let metrics = ObjectStoreCacheMetrics::new(); + metrics.cache_capacity.set(max_cache_size as f64); + metrics.cache_size.set(0.0); + + let cache = CacheBuilder::new(max_cache_size) + .time_to_live(ttl) + .build(); + + Self { cache: Arc::new(RwLock::new(cache)), object_store, - }) + min_fetch_size, + max_cache_size, + base_path, + metrics, + } } - /// Get an object from the cache or object store - pub async fn get(&self, path: &Path) -> Result { - // Try to get from cache first - if let Some(entry) = self.cache.read().await.get(path) { - return Ok(entry.value().clone()); + /// Get a range of data from the cache or object store + pub async fn get_range( + &self, + location: &Path, + range: Range, + ) -> Result { + debug!("{location}-{range:?} get_range"); + + // Expand the range to the next min_fetch_size (+ alignment) + let start_chunk = range.start / self.min_fetch_size as usize; + let end_chunk = (range.end - 1) / self.min_fetch_size as usize; + + let mut result = BytesMut::with_capacity( + (end_chunk.saturating_sub(start_chunk) + 1) * self.min_fetch_size as usize, + ); + + for chunk in start_chunk..=end_chunk { + let chunk_range = (chunk * self.min_fetch_size as usize) + ..((chunk + 1) * self.min_fetch_size as usize); + + let key = CacheKey { + path: location.to_owned(), + range: chunk_range.clone(), + }; + + let chunk_data = match self.cache.read().await.get(&key) { + Some(data) => { + debug!("Cache hit for {key:?}"); + self.metrics.cache_hits.increment(1); + data.clone() + } + None => { + debug!("Cache miss for {key:?}, fetching from object store"); + self.metrics.cache_misses.increment(1); + let start = Instant::now(); + let data = self.object_store.get_range(location, chunk_range.clone()).await?; + self.metrics.cache_read_latency.increment(start.elapsed().as_secs_f64()); + self.cache.write().await.insert(key, data.clone()); + data + } + }; + + result.extend_from_slice(&chunk_data); } - // If not in cache, get from object store - let bytes = self.object_store.get(path).await?.bytes().await?; + // Trim the result to match the requested range + let offset = range.start - start_chunk * self.min_fetch_size as usize; + result.advance(offset); + result.truncate(range.end - range.start); - // Store in cache - self.cache.write().await.insert(path.clone(), bytes.clone()); - - Ok(bytes) + debug!("{location}-{range:?} return"); + Ok(result.into()) } - /// Put an object into both cache and object store - pub async fn put(&self, path: &Path, bytes: Bytes) -> Result<()> { + /// Put data into both cache and object store + pub async fn put(&self, location: &Path, bytes: Bytes) -> Result<()> { // Store in object store - self.object_store.put(path, bytes.clone().into()).await?; - + self.object_store.put(location, bytes.clone().into()).await?; + // Store in cache - self.cache.write().await.insert(path.clone(), bytes); - + let key = CacheKey { + path: location.to_owned(), + range: 0..bytes.len(), + }; + self.cache.write().await.insert(key, bytes); + Ok(()) } - /// Remove an object from both cache and object store - pub async fn remove(&self, path: &Path) -> Result<()> { + /// Remove data from both cache and object store + pub async fn remove(&self, location: &Path) -> Result<()> { // Remove from object store - self.object_store.delete(path).await?; - + self.object_store.delete(location).await?; + // Remove from cache - self.cache.write().await.remove(path); - + let key = CacheKey { + path: location.to_owned(), + range: 0..usize::MAX, + }; + self.cache.write().await.remove(&key); + Ok(()) } } +impl Debug for ObjectStoreCache { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ObjectStoreCache") + .field("min_fetch_size", &self.min_fetch_size) + .field("max_cache_size", &self.max_cache_size) + .field("base_path", &self.base_path) + .finish() + } +} + #[cfg(test)] mod tests { + use super::*; use object_store::memory::InMemory; use tempfile::tempdir; - use super::*; - #[tokio::test] async fn test_object_store_cache() -> Result<()> { let temp_dir = tempdir()?; @@ -79,21 +221,24 @@ mod tests { let cache = ObjectStoreCache::new( object_store, temp_dir.path().to_path_buf(), - 1024 * 1024, // 1MB cache - )?; + DEFAULT_MIN_FETCH_SIZE, + DEFAULT_CACHE_CAPACITY, + DEFAULT_CACHE_ENTRY_TTL, + ); let path = Path::from("test.txt"); let data = Bytes::from("test data"); - // Test put and get + // Test put and get_range cache.put(&path, data.clone()).await?; - let retrieved = cache.get(&path).await?; + let retrieved = cache.get_range(&path, 0..data.len()).await?; assert_eq!(retrieved, data); // Test remove cache.remove(&path).await?; - assert!(cache.get(&path).await.is_err()); + assert!(cache.get_range(&path, 0..data.len()).await.is_err()); Ok(()) } } + diff --git a/src/object_store_cache_in_memory.rs b/src/object_store_cache_in_memory.rs new file mode 100644 index 00000000..6e99950e --- /dev/null +++ b/src/object_store_cache_in_memory.rs @@ -0,0 +1,95 @@ +use std::{path::PathBuf, sync::Arc}; + +use anyhow::Result; +use bytes::Bytes; +use foyer::{Cache, CacheBuilder}; +use object_store::{ObjectStore, path::Path}; +use tokio::sync::RwLock; + +/// A hybrid cache implementation for object store using foyer +pub struct ObjectStoreCache { + cache: Arc>>, + object_store: Arc, +} + +impl ObjectStoreCache { + /// Create a new ObjectStoreCache instance (in-memory cache) + pub fn new( + object_store: Arc, + capacity: usize, + ) -> Result { + let cache = CacheBuilder::new(capacity).build(); + Ok(Self { + cache: Arc::new(RwLock::new(cache)), + object_store, + }) + } + + /// Get an object from the cache or object store + pub async fn get(&self, path: &Path) -> Result { + // Try to get from cache first + if let Some(entry) = self.cache.read().await.get(path) { + return Ok(entry.value().clone()); + } + + // If not in cache, get from object store + let bytes = self.object_store.get(path).await?.bytes().await?; + + // Store in cache + self.cache.write().await.insert(path.clone(), bytes.clone()); + + Ok(bytes) + } + + /// Put an object into both cache and object store + pub async fn put(&self, path: &Path, bytes: Bytes) -> Result<()> { + // Store in object store + self.object_store.put(path, bytes.clone().into()).await?; + + // Store in cache + self.cache.write().await.insert(path.clone(), bytes); + + Ok(()) + } + + /// Remove an object from both cache and object store + pub async fn remove(&self, path: &Path) -> Result<()> { + // Remove from object store + self.object_store.delete(path).await?; + + // Remove from cache + self.cache.write().await.remove(path); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use object_store::memory::InMemory; + + use super::*; + + #[tokio::test] + async fn test_object_store_cache() -> Result<()> { + let object_store = Arc::new(InMemory::new()); + let cache = ObjectStoreCache::new( + object_store, + 1024 * 1024, // 1MB cache + )?; + + let path = Path::from("test.txt"); + let data = Bytes::from("test data"); + + // Test put and get + cache.put(&path, data.clone()).await?; + let retrieved = cache.get(&path).await?; + assert_eq!(retrieved, data); + + // Test remove + cache.remove(&path).await?; + assert!(cache.get(&path).await.is_err()); + + Ok(()) + } +} From df055162a575b80e24d03812ce61d9fdeaaaa008 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 2 Jun 2025 00:01:15 +0100 Subject: [PATCH 17/33] a file based implementation --- Cargo.lock | 11 ++++++++ Cargo.toml | 1 + src/object_store_cache.rs | 53 +++++++++++++++++++++------------------ 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d01efa5c..68a5ebe6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4551,6 +4551,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5" +dependencies = [ + "ahash 0.8.11", + "portable-atomic", +] + [[package]] name = "mime" version = "0.3.17" @@ -6886,6 +6896,7 @@ dependencies = [ "futures", "lazy_static", "log 0.4.27", + "metrics", "object_store 0.12.1", "opentelemetry", "opentelemetry-otlp", diff --git a/Cargo.toml b/Cargo.toml index ac48c971..901e7f4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ aws-sdk-s3 = "1.3.0" url = "2.5.4" datafusion-common = "46.0.0" tokio-cron-scheduler = "0.10" +metrics = "0.24.2" [dev-dependencies] serial_test = "3.2.0" diff --git a/src/object_store_cache.rs b/src/object_store_cache.rs index eccbbf79..a60518dc 100644 --- a/src/object_store_cache.rs +++ b/src/object_store_cache.rs @@ -1,4 +1,5 @@ -use std::{path::PathBuf, sync::Arc}; +use std::path::PathBuf; +use std::sync::Arc; use std::fmt::{Debug, Formatter}; use std::ops::Range; use std::time::{Duration, Instant}; @@ -6,12 +7,15 @@ use std::fs::remove_dir_all; use std::io; use anyhow::Result; -use bytes::{Bytes, BytesMut}; +use bytes::{Bytes, BytesMut, Buf}; use foyer::{Cache, CacheBuilder}; -use object_store::{ObjectStore, path::Path, PutPayload}; +use object_store::{ObjectStore, path::Path}; use tokio::sync::RwLock; use tracing::{debug, error, warn}; -use metrics::{counter, describe_counter, describe_histogram, gauge, histogram, Counter, Gauge}; +use metrics::{ + counter, describe_counter, describe_histogram, gauge, describe_gauge, + Counter, Gauge, Histogram, +}; /// Constants for cache configuration pub const DEFAULT_MIN_FETCH_SIZE: u64 = 1024 * 1024; // 1 MiB @@ -22,7 +26,7 @@ pub const DEFAULT_CACHE_ENTRY_TTL: Duration = Duration::from_secs(3 * 60); #[derive(Clone, Hash, Eq, PartialEq)] pub struct CacheKey { path: Path, - range: Range, + range: Range, } impl Debug for CacheKey { @@ -50,8 +54,8 @@ pub struct ObjectStoreCacheMetrics { cache_evictions: Counter, cache_size: Gauge, cache_capacity: Gauge, - cache_read_latency: Counter, - cache_write_latency: Counter, + cache_read_latency: Histogram, + cache_write_latency: Histogram, } impl ObjectStoreCacheMetrics { @@ -70,8 +74,8 @@ impl ObjectStoreCacheMetrics { cache_evictions: counter!("object_store_cache_evictions"), cache_size: gauge!("object_store_cache_size"), cache_capacity: gauge!("object_store_cache_capacity"), - cache_read_latency: counter!("object_store_cache_read_latency"), - cache_write_latency: counter!("object_store_cache_write_latency"), + cache_read_latency: metrics::histogram!("object_store_cache_read_latency"), + cache_write_latency: metrics::histogram!("object_store_cache_write_latency"), } } } @@ -99,8 +103,7 @@ impl ObjectStoreCache { metrics.cache_capacity.set(max_cache_size as f64); metrics.cache_size.set(0.0); - let cache = CacheBuilder::new(max_cache_size) - .time_to_live(ttl) + let cache = CacheBuilder::new(max_cache_size.try_into().unwrap()) .build(); Self { @@ -117,21 +120,21 @@ impl ObjectStoreCache { pub async fn get_range( &self, location: &Path, - range: Range, + range: Range, ) -> Result { debug!("{location}-{range:?} get_range"); // Expand the range to the next min_fetch_size (+ alignment) - let start_chunk = range.start / self.min_fetch_size as usize; - let end_chunk = (range.end - 1) / self.min_fetch_size as usize; + let start_chunk = (range.start / self.min_fetch_size) as usize; + let end_chunk = ((range.end - 1) / self.min_fetch_size) as usize; let mut result = BytesMut::with_capacity( (end_chunk.saturating_sub(start_chunk) + 1) * self.min_fetch_size as usize, ); for chunk in start_chunk..=end_chunk { - let chunk_range = (chunk * self.min_fetch_size as usize) - ..((chunk + 1) * self.min_fetch_size as usize); + let chunk_range = (chunk as u64 * self.min_fetch_size) + ..((chunk as u64 + 1) * self.min_fetch_size); let key = CacheKey { path: location.to_owned(), @@ -139,17 +142,17 @@ impl ObjectStoreCache { }; let chunk_data = match self.cache.read().await.get(&key) { - Some(data) => { + Some(entry) => { debug!("Cache hit for {key:?}"); self.metrics.cache_hits.increment(1); - data.clone() + entry.value().clone() } None => { debug!("Cache miss for {key:?}, fetching from object store"); self.metrics.cache_misses.increment(1); let start = Instant::now(); let data = self.object_store.get_range(location, chunk_range.clone()).await?; - self.metrics.cache_read_latency.increment(start.elapsed().as_secs_f64()); + self.metrics.cache_read_latency.record(start.elapsed().as_secs_f64()); self.cache.write().await.insert(key, data.clone()); data } @@ -159,9 +162,9 @@ impl ObjectStoreCache { } // Trim the result to match the requested range - let offset = range.start - start_chunk * self.min_fetch_size as usize; + let offset = (range.start - start_chunk as u64 * self.min_fetch_size) as usize; result.advance(offset); - result.truncate(range.end - range.start); + result.truncate((range.end - range.start) as usize); debug!("{location}-{range:?} return"); Ok(result.into()) @@ -175,7 +178,7 @@ impl ObjectStoreCache { // Store in cache let key = CacheKey { path: location.to_owned(), - range: 0..bytes.len(), + range: 0..bytes.len() as u64, }; self.cache.write().await.insert(key, bytes); @@ -190,7 +193,7 @@ impl ObjectStoreCache { // Remove from cache let key = CacheKey { path: location.to_owned(), - range: 0..usize::MAX, + range: 0..u64::MAX, }; self.cache.write().await.remove(&key); @@ -231,12 +234,12 @@ mod tests { // Test put and get_range cache.put(&path, data.clone()).await?; - let retrieved = cache.get_range(&path, 0..data.len()).await?; + let retrieved = cache.get_range(&path, 0..data.len() as u64).await?; assert_eq!(retrieved, data); // Test remove cache.remove(&path).await?; - assert!(cache.get_range(&path, 0..data.len()).await.is_err()); + assert!(cache.get_range(&path, 0..data.len() as u64).await.is_err()); Ok(()) } From 23156922f072dfc0824b3a0c9a79719fe6715679 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 2 Jun 2025 22:11:31 +0100 Subject: [PATCH 18/33] fininishing up --- src/object_store_cache.rs | 113 +++++++++++----------------- src/object_store_cache_in_memory.rs | 7 +- 2 files changed, 47 insertions(+), 73 deletions(-) diff --git a/src/object_store_cache.rs b/src/object_store_cache.rs index a60518dc..f9a45f97 100644 --- a/src/object_store_cache.rs +++ b/src/object_store_cache.rs @@ -1,31 +1,28 @@ -use std::path::PathBuf; -use std::sync::Arc; -use std::fmt::{Debug, Formatter}; -use std::ops::Range; -use std::time::{Duration, Instant}; -use std::fs::remove_dir_all; -use std::io; +use std::{ + fmt::{Debug, Formatter}, + ops::Range, + path::PathBuf, + sync::Arc, + time::{Duration, Instant}, +}; use anyhow::Result; -use bytes::{Bytes, BytesMut, Buf}; +use bytes::{Buf, Bytes, BytesMut}; use foyer::{Cache, CacheBuilder}; +use metrics::{Counter, Gauge, Histogram, counter, describe_counter, describe_gauge, describe_histogram, gauge}; use object_store::{ObjectStore, path::Path}; use tokio::sync::RwLock; -use tracing::{debug, error, warn}; -use metrics::{ - counter, describe_counter, describe_histogram, gauge, describe_gauge, - Counter, Gauge, Histogram, -}; +use tracing::debug; /// Constants for cache configuration pub const DEFAULT_MIN_FETCH_SIZE: u64 = 1024 * 1024; // 1 MiB pub const DEFAULT_CACHE_CAPACITY: u64 = 1024 * 1024 * 1024; // 1 GiB -pub const DEFAULT_CACHE_ENTRY_TTL: Duration = Duration::from_secs(3 * 60); + /// Cache key that includes both path and range information #[derive(Clone, Hash, Eq, PartialEq)] pub struct CacheKey { - path: Path, + path: Path, range: Range, } @@ -37,24 +34,19 @@ impl Debug for CacheKey { impl CacheKey { fn as_filename(&self) -> String { - format!( - "{}-{}-{}", - self.path.to_string().replace('/', "_"), - self.range.start, - self.range.end - ) + format!("{}-{}-{}", self.path.to_string().replace('/', "_"), self.range.start, self.range.end) } } /// Metrics for the object store cache #[derive(Clone)] pub struct ObjectStoreCacheMetrics { - cache_hits: Counter, - cache_misses: Counter, - cache_evictions: Counter, - cache_size: Gauge, - cache_capacity: Gauge, - cache_read_latency: Histogram, + cache_hits: Counter, + cache_misses: Counter, + cache_evictions: Counter, + cache_size: Gauge, + cache_capacity: Gauge, + cache_read_latency: Histogram, cache_write_latency: Histogram, } @@ -69,42 +61,35 @@ impl ObjectStoreCacheMetrics { describe_histogram!("object_store_cache_write_latency", "Cache write latency in seconds"); Self { - cache_hits: counter!("object_store_cache_hits"), - cache_misses: counter!("object_store_cache_misses"), - cache_evictions: counter!("object_store_cache_evictions"), - cache_size: gauge!("object_store_cache_size"), - cache_capacity: gauge!("object_store_cache_capacity"), - cache_read_latency: metrics::histogram!("object_store_cache_read_latency"), + cache_hits: counter!("object_store_cache_hits"), + cache_misses: counter!("object_store_cache_misses"), + cache_evictions: counter!("object_store_cache_evictions"), + cache_size: gauge!("object_store_cache_size"), + cache_capacity: gauge!("object_store_cache_capacity"), + cache_read_latency: metrics::histogram!("object_store_cache_read_latency"), cache_write_latency: metrics::histogram!("object_store_cache_write_latency"), } } } -/// A hybrid cache implementation for object store using foyer +/// A hybrid cache implementation for object store pub struct ObjectStoreCache { - cache: Arc>>, - object_store: Arc, + cache: Arc>>, + object_store: Arc, min_fetch_size: u64, max_cache_size: u64, - base_path: PathBuf, - metrics: ObjectStoreCacheMetrics, + base_path: PathBuf, + metrics: ObjectStoreCacheMetrics, } impl ObjectStoreCache { /// Create a new ObjectStoreCache instance - pub fn new( - object_store: Arc, - base_path: PathBuf, - min_fetch_size: u64, - max_cache_size: u64, - ttl: Duration, - ) -> Self { + pub fn new(object_store: Arc, base_path: PathBuf, min_fetch_size: u64, max_cache_size: u64) -> Self { let metrics = ObjectStoreCacheMetrics::new(); metrics.cache_capacity.set(max_cache_size as f64); metrics.cache_size.set(0.0); - let cache = CacheBuilder::new(max_cache_size.try_into().unwrap()) - .build(); + let cache = CacheBuilder::new(max_cache_size.try_into().unwrap()).build(); Self { cache: Arc::new(RwLock::new(cache)), @@ -117,27 +102,20 @@ impl ObjectStoreCache { } /// Get a range of data from the cache or object store - pub async fn get_range( - &self, - location: &Path, - range: Range, - ) -> Result { + pub async fn get_range(&self, location: &Path, range: Range) -> Result { debug!("{location}-{range:?} get_range"); - + // Expand the range to the next min_fetch_size (+ alignment) let start_chunk = (range.start / self.min_fetch_size) as usize; let end_chunk = ((range.end - 1) / self.min_fetch_size) as usize; - let mut result = BytesMut::with_capacity( - (end_chunk.saturating_sub(start_chunk) + 1) * self.min_fetch_size as usize, - ); + let mut result = BytesMut::with_capacity((end_chunk.saturating_sub(start_chunk) + 1) * self.min_fetch_size as usize); for chunk in start_chunk..=end_chunk { - let chunk_range = (chunk as u64 * self.min_fetch_size) - ..((chunk as u64 + 1) * self.min_fetch_size); + let chunk_range = (chunk as u64 * self.min_fetch_size)..((chunk as u64 + 1) * self.min_fetch_size); let key = CacheKey { - path: location.to_owned(), + path: location.to_owned(), range: chunk_range.clone(), }; @@ -174,14 +152,14 @@ impl ObjectStoreCache { pub async fn put(&self, location: &Path, bytes: Bytes) -> Result<()> { // Store in object store self.object_store.put(location, bytes.clone().into()).await?; - + // Store in cache let key = CacheKey { - path: location.to_owned(), + path: location.to_owned(), range: 0..bytes.len() as u64, }; self.cache.write().await.insert(key, bytes); - + Ok(()) } @@ -189,14 +167,14 @@ impl ObjectStoreCache { pub async fn remove(&self, location: &Path) -> Result<()> { // Remove from object store self.object_store.delete(location).await?; - + // Remove from cache let key = CacheKey { - path: location.to_owned(), + path: location.to_owned(), range: 0..u64::MAX, }; self.cache.write().await.remove(&key); - + Ok(()) } } @@ -213,10 +191,11 @@ impl Debug for ObjectStoreCache { #[cfg(test)] mod tests { - use super::*; use object_store::memory::InMemory; use tempfile::tempdir; + use super::*; + #[tokio::test] async fn test_object_store_cache() -> Result<()> { let temp_dir = tempdir()?; @@ -226,7 +205,6 @@ mod tests { temp_dir.path().to_path_buf(), DEFAULT_MIN_FETCH_SIZE, DEFAULT_CACHE_CAPACITY, - DEFAULT_CACHE_ENTRY_TTL, ); let path = Path::from("test.txt"); @@ -244,4 +222,3 @@ mod tests { Ok(()) } } - diff --git a/src/object_store_cache_in_memory.rs b/src/object_store_cache_in_memory.rs index 6e99950e..fd79008b 100644 --- a/src/object_store_cache_in_memory.rs +++ b/src/object_store_cache_in_memory.rs @@ -6,7 +6,7 @@ use foyer::{Cache, CacheBuilder}; use object_store::{ObjectStore, path::Path}; use tokio::sync::RwLock; -/// A hybrid cache implementation for object store using foyer +/// A hybrid cache implementation for object store pub struct ObjectStoreCache { cache: Arc>>, object_store: Arc, @@ -14,10 +14,7 @@ pub struct ObjectStoreCache { impl ObjectStoreCache { /// Create a new ObjectStoreCache instance (in-memory cache) - pub fn new( - object_store: Arc, - capacity: usize, - ) -> Result { + pub fn new(object_store: Arc, capacity: usize) -> Result { let cache = CacheBuilder::new(capacity).build(); Ok(Self { cache: Arc::new(RwLock::new(cache)), From f521af1ed7da5991f2dc8501ed50114337048407 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 6 Jun 2025 00:41:15 +0100 Subject: [PATCH 19/33] refactored hybrid cache --- Cargo.lock | 90 ++-- Cargo.toml | 13 +- src/delta.rs | 414 ++++++++++++++++ src/lib.rs | 4 +- src/obj_store.rs | 700 ++++++++++++++++++++++++++++ src/object_store_cache.rs | 224 --------- src/object_store_cache_in_memory.rs | 92 ---- 7 files changed, 1180 insertions(+), 357 deletions(-) create mode 100644 src/delta.rs create mode 100644 src/obj_store.rs delete mode 100644 src/object_store_cache.rs delete mode 100644 src/object_store_cache_in_memory.rs diff --git a/Cargo.lock b/Cargo.lock index 68a5ebe6..be2e9ccf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2165,7 +2165,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log 0.4.27", - "object_store 0.11.2", + "object_store", "parking_lot 0.12.3", "parquet", "rand 0.8.5", @@ -2217,7 +2217,7 @@ dependencies = [ "datafusion-physical-plan", "futures", "log 0.4.27", - "object_store 0.11.2", + "object_store", "tokio", ] @@ -2236,7 +2236,7 @@ dependencies = [ "indexmap 2.9.0", "libc", "log 0.4.27", - "object_store 0.11.2", + "object_store", "parquet", "paste", "recursive", @@ -2280,7 +2280,7 @@ dependencies = [ "glob", "itertools 0.14.0", "log 0.4.27", - "object_store 0.11.2", + "object_store", "rand 0.8.5", "tokio", "tokio-util", @@ -2307,7 +2307,7 @@ dependencies = [ "datafusion-expr", "futures", "log 0.4.27", - "object_store 0.11.2", + "object_store", "parking_lot 0.12.3", "rand 0.8.5", "tempfile", @@ -2626,7 +2626,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-proto-common", - "object_store 0.11.2", + "object_store", "prost", ] @@ -2684,7 +2684,7 @@ dependencies = [ "home", "indexmap 2.9.0", "itertools 0.13.0", - "object_store 0.11.2", + "object_store", "parquet", "reqwest", "roaring", @@ -2719,7 +2719,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78889f4005974b848f130fa5dedae81987f1bc93b107291ea87d900c93b6c3bb" dependencies = [ "deltalake-aws", + "deltalake-azure", "deltalake-core", + "deltalake-gcp", ] [[package]] @@ -2740,7 +2742,7 @@ dependencies = [ "deltalake-core", "futures", "maplit", - "object_store 0.11.2", + "object_store", "regex 1.11.1", "thiserror 2.0.12", "tokio", @@ -2749,6 +2751,24 @@ dependencies = [ "uuid", ] +[[package]] +name = "deltalake-azure" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfbea4786321ebd88e083ec74ce513ec7fcad9ddc880b611770dee012652567" +dependencies = [ + "async-trait", + "bytes", + "deltalake-core", + "futures", + "object_store", + "regex 1.11.1", + "thiserror 2.0.12", + "tokio", + "tracing", + "url", +] + [[package]] name = "deltalake-core" version = "0.25.0" @@ -2793,7 +2813,7 @@ dependencies = [ "num-bigint", "num-traits", "num_cpus", - "object_store 0.11.2", + "object_store", "parking_lot 0.12.3", "parquet", "percent-encoding", @@ -2814,6 +2834,24 @@ dependencies = [ "z85", ] +[[package]] +name = "deltalake-gcp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa80de5b3e9e53eb9a98d976ac1d64a70b798a73d63cdd83497cc317a5063602" +dependencies = [ + "async-trait", + "bytes", + "deltalake-core", + "futures", + "object_store", + "regex 1.11.1", + "thiserror 2.0.12", + "tokio", + "tracing", + "url", +] + [[package]] name = "der" version = "0.6.1" @@ -3227,6 +3265,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ed2316785e80137c7b91bb74dab1dc1967c3272df05825397b73ae8fc527041" dependencies = [ "ahash 0.8.11", + "bincode", "bytes", "cfg-if", "itertools 0.14.0", @@ -3234,6 +3273,7 @@ dependencies = [ "mixtrics", "parking_lot 0.12.3", "pin-project", + "serde", "thiserror 2.0.12", "tokio", ] @@ -3301,6 +3341,7 @@ dependencies = [ "paste", "pin-project", "rand 0.9.0", + "serde", "thiserror 2.0.12", "tokio", "tracing", @@ -4796,6 +4837,7 @@ dependencies = [ "bytes", "chrono", "futures", + "httparse", "humantime", "hyper 1.6.0", "itertools 0.13.0", @@ -4806,6 +4848,7 @@ dependencies = [ "rand 0.8.5", "reqwest", "ring", + "rustls-pemfile 2.2.0", "serde", "serde_json", "snafu", @@ -4815,30 +4858,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "object_store" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d94ac16b433c0ccf75326388c893d2835ab7457ea35ab8ba5d745c053ef5fa16" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures", - "http 1.3.1", - "humantime", - "itertools 0.14.0", - "parking_lot 0.12.3", - "percent-encoding", - "thiserror 2.0.12", - "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -5111,7 +5130,7 @@ dependencies = [ "lz4_flex", "num", "num-bigint", - "object_store 0.11.2", + "object_store", "paste", "seq-macro", "simdutf8", @@ -6892,12 +6911,13 @@ dependencies = [ "deltalake", "dotenv", "env_logger", + "flate2", "foyer", "futures", "lazy_static", "log 0.4.27", "metrics", - "object_store 0.12.1", + "object_store", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", diff --git a/Cargo.toml b/Cargo.toml index 901e7f4e..394e559f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,8 +5,8 @@ edition = "2024" [dependencies] tokio = { version = "1.43", features = ["full"] } -foyer = "0.17.0" -object_store = "0.12.1" +foyer = {version ="0.17.0",features=["serde"]} +object_store = { version = "0.11.2", features = ["aws", "azure", "gcp"] } datafusion = "46.0.0" arrow = "54.2.0" uuid = { version = "1.13", features = ["v4", "serde"] } @@ -20,7 +20,7 @@ log = "0.4.25" color-eyre = "0.6.3" arrow-schema = "54.1.0" regex = "1.11.1" -deltalake = { version = "0.25.0", features = ["datafusion", "s3"] } +deltalake = { version = "0.25.0", features = ["datafusion", "s3","azure", "gcs",] } delta_kernel = { version = "0.8.0", features = [ "arrow-conversion", "default-engine", @@ -70,6 +70,7 @@ url = "2.5.4" datafusion-common = "46.0.0" tokio-cron-scheduler = "0.10" metrics = "0.24.2" +flate2 = "1.1.1" [dev-dependencies] serial_test = "3.2.0" @@ -78,5 +79,9 @@ scopeguard = "1.2.0" rand = "0.8.5" [features] -default = [] +default = ["s3", "azure", "gcs"] +s3 = ["deltalake/s3", "object_store/aws"] +azure = ["deltalake/azure", "object_store/azure"] +gcs = ["deltalake/gcs", "object_store/gcp"] test = [] + diff --git a/src/delta.rs b/src/delta.rs new file mode 100644 index 00000000..ead92adf --- /dev/null +++ b/src/delta.rs @@ -0,0 +1,414 @@ +use std::sync::Arc; +use std::time::Duration; +use std::collections::HashMap; + +use deltalake::{ + DeltaTable, DeltaTableBuilder, DeltaTableError, + operations::{create::CreateBuilder, write::WriteBuilder}, + arrow::record_batch::RecordBatch, + logstore::LogStore, + storage::ObjectStoreRef, +}; +use object_store::{ + aws::AmazonS3Builder, + azure::MicrosoftAzureBuilder, + gcp::GoogleCloudStorageBuilder, + local::LocalFileSystem, + memory::InMemory, + ObjectStore, +}; +use tokio; +use url::Url; +use crate::obj_store::{CacheMetrics, DeltaCacheBuilder, DeltaCacheConfig, DeltaCachedObjectStore}; + +/// Helper struct for creating Delta tables with caching +pub struct CachedDeltaTableBuilder { + table_uri: String, + cache_config: Option, + storage_options: HashMap, +} + +impl CachedDeltaTableBuilder { + pub fn new>(table_uri: S) -> Self { + Self { + table_uri: table_uri.into(), + cache_config: None, + storage_options: HashMap::new(), + } + } + + /// Enable caching with custom configuration + pub fn with_cache_config(mut self, config: DeltaCacheConfig) -> Self { + self.cache_config = Some(config); + self + } + + /// Enable caching with default configuration + pub fn with_cache(mut self) -> Self { + self.cache_config = Some(DeltaCacheConfig::default()); + self + } + + /// Add storage options (AWS credentials, etc.) + pub fn with_storage_options(mut self, options: HashMap) -> Self { + self.storage_options = options; + self + } + + /// Add a single storage option + pub fn with_storage_option, V: Into>( + mut self, + key: K, + value: V + ) -> Self { + self.storage_options.insert(key.into(), value.into()); + self + } + + /// Build the Delta table with caching + pub async fn build(self) -> Result { + let base_store = self.create_base_object_store().await?; + + let final_store: ObjectStoreRef = if let Some(cache_config) = self.cache_config { + // Wrap with cache + let cached_store = DeltaCacheBuilder::new() + .with_memory_capacity(cache_config.memory_capacity) + .with_disk_capacity(cache_config.disk_capacity) + .with_disk_path(cache_config.disk_cache_dir) + .with_ttl(Duration::from_secs(cache_config.ttl_seconds)) + .with_compression(cache_config.compression_level) + .enable_metrics(cache_config.enable_metrics) + .enable_cache_warming(cache_config.enable_cache_warming) + .cache_transaction_logs(cache_config.cache_transaction_logs) + .cache_parquet_metadata(cache_config.cache_parquet_metadata) + .cache_checkpoints(cache_config.cache_checkpoints) + .build(base_store) + .await + .map_err(|e| DeltaTableError::ObjectStore { source: e })?; + + cached_store + } else { + base_store + }; + + // Create Delta table with the (potentially cached) object store + DeltaTableBuilder::from_uri(&self.table_uri) + .with_object_store(final_store) + .load() + .await + } + + /// Create the base object store based on URI scheme + async fn create_base_object_store(&self) -> Result { + let uri = Url::parse(&self.table_uri) + .map_err(|e| DeltaTableError::Generic(format!("Invalid URI: {}", e)))?; + + match uri.scheme() { + "s3" | "s3a" => { + let mut builder = AmazonS3Builder::new(); + + // Apply storage options + for (key, value) in &self.storage_options { + match key.as_str() { + "AWS_ACCESS_KEY_ID" => builder = builder.with_access_key_id(value), + "AWS_SECRET_ACCESS_KEY" => builder = builder.with_secret_access_key(value), + "AWS_REGION" => builder = builder.with_region(value), + "AWS_ENDPOINT" => builder = builder.with_endpoint(value), + "AWS_BUCKET_NAME" => builder = builder.with_bucket_name(value), + _ => {} + } + } + + let bucket = uri.host_str() + .ok_or_else(|| DeltaTableError::Generic("No bucket in S3 URI".to_string()))?; + + Ok(Arc::new(builder.with_bucket_name(bucket).build() + .map_err(|e| DeltaTableError::ObjectStore { source: e })?)) + } + + "abfs" | "abfss" => { + let mut builder = MicrosoftAzureBuilder::new(); + + // Apply storage options + for (key, value) in &self.storage_options { + match key.as_str() { + "AZURE_STORAGE_ACCOUNT_NAME" => builder = builder.with_account(value), + "AZURE_STORAGE_ACCOUNT_KEY" => builder = builder.with_access_key(value), + "AZURE_STORAGE_SAS_TOKEN" => builder = builder.with_sas_token(value), + "AZURE_STORAGE_CONTAINER_NAME" => builder = builder.with_container_name(value), + _ => {} + } + } + + Ok(Arc::new(builder.build() + .map_err(|e| DeltaTableError::ObjectStore { source: e })?)) + } + + "gs" => { + let mut builder = GoogleCloudStorageBuilder::new(); + + // Apply storage options + for (key, value) in &self.storage_options { + match key.as_str() { + "GOOGLE_SERVICE_ACCOUNT" => builder = builder.with_service_account_path(value), + "GOOGLE_SERVICE_ACCOUNT_KEY" => builder = builder.with_service_account_key(value), + "GOOGLE_BUCKET_NAME" => builder = builder.with_bucket_name(value), + _ => {} + } + } + + let bucket = uri.host_str() + .ok_or_else(|| DeltaTableError::Generic("No bucket in GCS URI".to_string()))?; + + Ok(Arc::new(builder.with_bucket_name(bucket).build() + .map_err(|e| DeltaTableError::ObjectStore { source: e })?)) + } + + "file" => { + let path = uri.to_file_path() + .map_err(|_| DeltaTableError::Generic("Invalid file path".to_string()))?; + Ok(Arc::new(LocalFileSystem::new_with_prefix(path) + .map_err(|e| DeltaTableError::ObjectStore { source: e })?)) + } + + "memory" => { + Ok(Arc::new(InMemory::new())) + } + + scheme => Err(DeltaTableError::Generic(format!("Unsupported scheme: {}", scheme))) + } + } +} + +/// Extension trait for DeltaTable to access cache metrics +pub trait DeltaTableCacheExt { + /// Get cache metrics if the table is using a cached store + async fn cache_metrics(&self) -> Option; + + /// Get access patterns if the table is using a cached store + async fn access_patterns(&self) -> Option>; +} + +impl DeltaTableCacheExt for DeltaTable { + async fn cache_metrics(&self) -> Option { + // Try to downcast the object store to our cached implementation + let store = self.object_store(); + if let Some(cached_store) = store.as_any().downcast_ref::() { + Some(cached_store.metrics().await) + } else { + None + } + } + + async fn access_patterns(&self) -> Option> { + let store = self.object_store(); + if let Some(cached_store) = store.as_any().downcast_ref::() { + Some(cached_store.get_access_patterns().await) + } else { + None + } + } +} + +/// Convenience functions for common Delta operations with caching +pub struct CachedDeltaOps; + +impl CachedDeltaOps { + /// Create a new Delta table with caching enabled + pub async fn create_table( + table_uri: &str, + schema: arrow::datatypes::SchemaRef, + cache_config: Option, + ) -> Result { + let mut builder = CachedDeltaTableBuilder::new(table_uri); + + if let Some(config) = cache_config { + builder = builder.with_cache_config(config); + } + + let table = builder.build().await?; + + // Create the table if it doesn't exist + CreateBuilder::new() + .with_object_store(table.object_store()) + .with_table_uri(table_uri) + .with_columns(schema.fields().iter().cloned()) + .await?; + + // Reload to get the created table + builder.build().await + } + + /// Open an existing Delta table with caching + pub async fn open_table( + table_uri: &str, + cache_config: Option, + storage_options: Option>, + ) -> Result { + let mut builder = CachedDeltaTableBuilder::new(table_uri); + + if let Some(config) = cache_config { + builder = builder.with_cache_config(config); + } + + if let Some(options) = storage_options { + builder = builder.with_storage_options(options); + } + + builder.build().await + } + + /// Write data to a Delta table with caching + pub async fn write_to_table( + table: &mut DeltaTable, + batches: Vec, + ) -> Result<(), DeltaTableError> { + WriteBuilder::new(table.object_store(), table.table_uri()) + .with_input_batches(batches)? + .await?; + + // Reload the table to see the new data + *table = table.load().await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Int32Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use std::sync::Arc; + use tempfile::TempDir; + + #[tokio::test] + async fn test_cached_delta_table_creation() { + let temp_dir = TempDir::new().unwrap(); + let table_uri = format!("file://{}", temp_dir.path().to_str().unwrap()); + + // Create schema + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ])); + + // Create cache config + let cache_config = DeltaCacheConfig { + memory_capacity: 64 * 1024 * 1024, // 64MB + disk_capacity: 128 * 1024 * 1024, // 128MB + disk_cache_dir: temp_dir.path().join("cache").to_str().unwrap().to_string(), + ttl_seconds: 300, // 5 minutes + enable_metrics: true, + ..Default::default() + }; + + // Create table with caching + let table = CachedDeltaOps::create_table( + &table_uri, + schema.clone(), + Some(cache_config), + ).await.unwrap(); + + // Verify the table was created + assert!(table.get_files_count() == 0); // New table, no data files yet + + // Check if cache metrics are available + if let Some(metrics) = table.cache_metrics().await { + println!("Cache metrics: {:?}", metrics); + } + } + + #[tokio::test] + async fn test_write_and_read_with_cache() { + let temp_dir = TempDir::new().unwrap(); + let table_uri = format!("file://{}", temp_dir.path().to_str().unwrap()); + + // Create schema + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ])); + + // Create table with cache + let mut table = CachedDeltaOps::create_table( + &table_uri, + schema.clone(), + Some(DeltaCacheConfig::default()), + ).await.unwrap(); + + // Create some test data + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])), + ], + ).unwrap(); + + // Write data + CachedDeltaOps::write_to_table(&mut table, vec![batch]).await.unwrap(); + + // Read data back (should hit cache on subsequent reads) + let files = table.get_files(); + assert!(!files.is_empty()); + + // Check cache metrics + if let Some(metrics) = table.cache_metrics().await { + println!("After write - Cache metrics: {:?}", metrics); + } + + // Read again to test cache hit + let _files_again = table.get_files(); + + if let Some(metrics) = table.cache_metrics().await { + println!("After second read - Cache metrics: {:?}", metrics); + assert!(metrics.total_requests > 0); + } + } +} + +// Example usage in your application +#[tokio::main] +async fn main() -> Result<(), Box> { + // Example 1: Simple cached Delta table + let cache_config = DeltaCacheConfig { + memory_capacity: 256 * 1024 * 1024, // 256MB + disk_capacity: 1024 * 1024 * 1024, // 1GB + disk_cache_dir: "/tmp/delta_cache".to_string(), + ttl_seconds: 3600, // 1 hour + enable_metrics: true, + enable_cache_warming: true, + ..Default::default() + }; + + let table = CachedDeltaTableBuilder::new("s3://my-bucket/my-table") + .with_cache_config(cache_config) + .with_storage_option("AWS_REGION", "us-west-2") + .with_storage_option("AWS_ACCESS_KEY_ID", "your-access-key") + .with_storage_option("AWS_SECRET_ACCESS_KEY", "your-secret-key") + .build() + .await?; + + println!("Table loaded with {} files", table.get_files_count()); + + // Check cache performance + if let Some(metrics) = table.cache_metrics().await { + println!("Cache hit rate: {:.2}%", metrics.hit_rate() * 100.0); + println!("Total requests: {}", metrics.total_requests); + println!("Cache hits: {}", metrics.hits); + println!("Cache misses: {}", metrics.misses); + } + + // Example 2: Monitor access patterns + if let Some(patterns) = table.access_patterns().await { + println!("Most accessed files:"); + let mut sorted_patterns: Vec<_> = patterns.iter().collect(); + sorted_patterns.sort_by(|a, b| b.1.cmp(a.1)); + + for (path, count) in sorted_patterns.iter().take(10) { + println!(" {}: {} accesses", path, count); + } + } + + Ok(()) +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 58a41bb5..d905801a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ // lib.rs - Export modules for use in tests pub mod batch_queue; pub mod database; -pub mod object_store_cache; -pub mod object_store_cache_in_memory; +pub mod obj_store; +pub mod delta; pub mod persistent_queue; diff --git a/src/obj_store.rs b/src/obj_store.rs new file mode 100644 index 00000000..daf54f45 --- /dev/null +++ b/src/obj_store.rs @@ -0,0 +1,700 @@ +use std::fmt::{Debug, Display}; +use std::ops::Range; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use bytes::Bytes; +use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; +use futures::stream::BoxStream; +use object_store::{ + path::Path, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, + ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult, +}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, info, warn, error}; +use tokio::sync::RwLock; +use std::collections::HashMap; + +/// Configuration for the Delta cache +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeltaCacheConfig { + /// Memory cache capacity in bytes + pub memory_capacity: usize, + /// Disk cache capacity in bytes + pub disk_capacity: usize, + /// Disk cache directory path + pub disk_cache_dir: String, + /// TTL for cached objects in seconds + pub ttl_seconds: u64, + /// Whether to cache transaction logs + pub cache_transaction_logs: bool, + /// Whether to cache parquet metadata + pub cache_parquet_metadata: bool, + /// Whether to cache checkpoint files + pub cache_checkpoints: bool, + /// Maximum object size to cache (in bytes) + pub max_object_size: usize, + /// Enable metrics collection + pub enable_metrics: bool, + /// Cache warming on startup + pub enable_cache_warming: bool, + /// Compression level for cached data (0-9, 0=no compression) + pub compression_level: u8, +} + +impl Default for DeltaCacheConfig { + fn default() -> Self { + Self { + memory_capacity: 256 * 1024 * 1024, // 256MB + disk_capacity: 1024 * 1024 * 1024, // 1GB + disk_cache_dir: "/tmp/delta_cache".to_string(), + ttl_seconds: 3600, // 1 hour + cache_transaction_logs: true, + cache_parquet_metadata: true, + cache_checkpoints: true, + max_object_size: 10 * 1024 * 1024, // 10MB max + enable_metrics: true, + enable_cache_warming: false, + compression_level: 3, // Light compression by default + } + } +} + +/// Cache metrics for monitoring +#[derive(Debug, Default, Clone)] +pub struct CacheMetrics { + pub hits: u64, + pub misses: u64, + pub evictions: u64, + pub errors: u64, + pub total_requests: u64, + pub cache_size_bytes: u64, +} + +impl CacheMetrics { + pub fn hit_rate(&self) -> f64 { + if self.total_requests == 0 { + 0.0 + } else { + self.hits as f64 / self.total_requests as f64 + } + } +} + +/// Enhanced cached object with metadata and compression +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CachedObject { + data: Vec, + cached_at: u64, + original_size: usize, + is_compressed: bool, + etag: Option, + last_modified: Option, +} + +impl CachedObject { + fn new(data: Vec, meta: &ObjectMeta, compression_level: u8) -> Self { + let original_size = data.len(); + let (final_data, is_compressed) = if compression_level > 0 && data.len() > 1024 { + match Self::compress(&data, compression_level) { + Ok(compressed) if compressed.len() < data.len() => (compressed, true), + _ => (data, false), + } + } else { + (data, false) + }; + + Self { + data: final_data, + cached_at: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + original_size, + is_compressed, + etag: meta.e_tag.clone(), + last_modified: meta.last_modified.map(|t| t.timestamp()), + } + } + + fn get_data(&self) -> Result, std::io::Error> { + if self.is_compressed { + Self::decompress(&self.data) + } else { + Ok(self.data.clone()) + } + } + + fn compress(data: &[u8], level: u8) -> Result, std::io::Error> { + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::Write; + + let mut encoder = GzEncoder::new(Vec::new(), Compression::new(level as u32)); + encoder.write_all(data)?; + encoder.finish() + } + + fn decompress(data: &[u8]) -> Result, std::io::Error> { + use flate2::read::GzDecoder; + use std::io::Read; + + let mut decoder = GzDecoder::new(data); + let mut decompressed = Vec::new(); + decoder.read_to_end(&mut decompressed)?; + Ok(decompressed) + } + + fn is_valid(&self, ttl_seconds: u64) -> bool { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + (now - self.cached_at) < ttl_seconds + } + + fn matches_meta(&self, meta: &ObjectMeta) -> bool { + // Check ETag if available + if let (Some(cached_etag), Some(meta_etag)) = (&self.etag, &meta.e_tag) { + return cached_etag == meta_etag; + } + + // Fallback to last modified time + if let (Some(cached_modified), Some(meta_modified)) = (self.last_modified, meta.last_modified) { + return cached_modified == meta_modified.timestamp(); + } + + // If no metadata available, assume valid (will rely on TTL) + true + } +} + +/// Delta-optimized object store cache wrapper +#[derive(Debug)] +pub struct DeltaCachedObjectStore { + inner: Arc, + cache: Arc>, + config: DeltaCacheConfig, + metrics: Arc>, + // Track frequently accessed paths for cache warming + access_patterns: Arc>>, +} + +impl DeltaCachedObjectStore { + /// Create a new cached object store + pub async fn new( + inner: Arc, + config: DeltaCacheConfig, + ) -> ObjectStoreResult { + // Build the hybrid cache + let cache = HybridCacheBuilder::new() + .memory(config.memory_capacity) + .storage(Engine::Large) + .with_device_options(DirectFsDeviceOptions::new(&config.disk_cache_dir) + .with_capacity(config.disk_capacity)) + .build() + .await + .map_err(|e| object_store::Error::Generic { + store: "DeltaCache", + source: Box::new(std::io::Error::other(e)), + })?; + + info!( + "Initialized Delta cache: memory={}MB, disk={}GB, dir={}, compression={}", + config.memory_capacity / (1024 * 1024), + config.disk_capacity / (1024 * 1024 * 1024), + config.disk_cache_dir, + if config.compression_level > 0 { "enabled" } else { "disabled" } + ); + + let store = Self { + inner, + cache: Arc::new(cache), + config, + metrics: Arc::new(RwLock::new(CacheMetrics::default())), + access_patterns: Arc::new(RwLock::new(HashMap::new())), + }; + + // Optionally warm the cache + if store.config.enable_cache_warming { + tokio::spawn({ + let store_clone = store.clone(); + async move { + if let Err(e) = store_clone.warm_cache().await { + warn!("Cache warming failed: {}", e); + } + } + }); + } + + Ok(store) + } + + /// Get current cache metrics + pub async fn metrics(&self) -> CacheMetrics { + self.metrics.read().await.clone() + } + + /// Warm the cache by pre-loading frequently accessed files + async fn warm_cache(&self) -> ObjectStoreResult<()> { + info!("Starting cache warming..."); + + // Focus on Delta log directory + let delta_log_prefix = Path::from("_delta_log"); + let mut stream = self.inner.list(Some(&delta_log_prefix)); + let mut warmed_count = 0; + + while let Some(meta_result) = futures::StreamExt::next(&mut stream).await { + let meta = meta_result?; + + // Only warm small, frequently accessed files + if meta.size <= (1024 * 1024) && self.should_cache(&meta.location) { + match self.inner.get(&meta.location).await { + Ok(result) => { + if let Ok(bytes) = result.bytes().await { + let cache_key = self.make_cache_key(&meta.location, None); + let cached_obj = CachedObject::new( + bytes.to_vec(), + &meta, + self.config.compression_level + ); + + if self.cache.insert(cache_key, cached_obj).await.is_ok() { + warmed_count += 1; + } + } + } + Err(e) => debug!("Failed to warm cache for {}: {}", meta.location, e), + } + } + } + + info!("Cache warming completed: {} files preloaded", warmed_count); + Ok(()) + } + + /// Enhanced path caching logic with more granular control + fn should_cache(&self, path: &Path) -> bool { + let path_str = path.as_ref(); + + // Always cache Delta log directory contents + if path_str.contains("_delta_log/") { + // Transaction logs + if path_str.ends_with(".json") && self.config.cache_transaction_logs { + return true; + } + // Checkpoint files + if path_str.ends_with(".checkpoint.parquet") && self.config.cache_checkpoints { + return true; + } + // Other Delta log files (like .crc files) + return true; + } + + // Parquet metadata files + if self.config.cache_parquet_metadata { + if path_str.ends_with(".parquet") + || path_str.contains("_metadata") + || path_str.ends_with("_common_metadata") { + return true; + } + } + + false + } + + /// Create cache key from path and optional range + fn make_cache_key(&self, path: &Path, range: Option<&GetRange>) -> String { + match range { + Some(GetRange::Bounded(r)) => format!("{}:{}:{}", path.as_ref(), r.start, r.end), + Some(GetRange::Offset(offset)) => format!("{}:{}:", path.as_ref(), offset), + Some(GetRange::Suffix(suffix)) => format!("{}:suffix:{}", path.as_ref(), suffix), + None => path.as_ref().to_string(), + } + } + + /// Update access patterns for analytics + async fn record_access(&self, path: &str) { + let mut patterns = self.access_patterns.write().await; + *patterns.entry(path.to_string()).or_insert(0) += 1; + } + + /// Update metrics + async fn update_metrics(&self, update_fn: F) + where + F: FnOnce(&mut CacheMetrics), + { + if self.config.enable_metrics { + let mut metrics = self.metrics.write().await; + update_fn(&mut *metrics); + } + } + + /// Enhanced cache retrieval with metadata validation + async fn get_with_cache( + &self, + location: &Path, + options: GetOptions, + ) -> ObjectStoreResult { + self.update_metrics(|m| m.total_requests += 1).await; + + // Check if we should cache this path + if !self.should_cache(location) { + debug!("Path not cacheable, delegating: {}", location); + return self.inner.get_opts(location, options).await; + } + + // Record access pattern + self.record_access(location.as_ref()).await; + + // For range requests, bypass cache for now (could be enhanced later) + if options.range.is_some() { + debug!("Range request, bypassing cache: {}", location); + return self.inner.get_opts(location, options).await; + } + + let cache_key = self.make_cache_key(location, None); + + // Try to get from cache first + if let Ok(Some(cached_entry)) = self.cache.get(&cache_key).await { + let cached_obj = cached_entry.value(); + + // Check if cache entry is still valid + if cached_obj.is_valid(self.config.ttl_seconds) { + // Get fresh metadata for validation + match self.inner.head(location).await { + Ok(meta) => { + // Validate cache against metadata + if cached_obj.matches_meta(&meta) { + debug!("Cache hit for: {}", location); + self.update_metrics(|m| m.hits += 1).await; + + // Decompress if needed + match cached_obj.get_data() { + Ok(data) => { + let bytes = Bytes::from(data); + return Ok(GetResult { + payload: GetResultPayload::Stream( + Box::pin(futures::stream::once(async { Ok(bytes) })) + ), + meta: meta.clone(), + range: 0..meta.size, + attributes: Default::default(), + }); + } + Err(e) => { + error!("Failed to decompress cached data for {}: {}", location, e); + // Fall through to cache miss + } + } + } else { + debug!("Cache invalidated due to metadata mismatch: {}", location); + // Remove stale entry + let _ = self.cache.remove(&cache_key); + } + } + Err(e) => { + debug!("Failed to get metadata for cache validation: {} - {}", location, e); + // Use cached data anyway if metadata lookup fails + if let Ok(data) = cached_obj.get_data() { + self.update_metrics(|m| m.hits += 1).await; + let bytes = Bytes::from(data); + return Ok(GetResult { + payload: GetResultPayload::Stream( + Box::pin(futures::stream::once(async { Ok(bytes) })) + ), + meta: ObjectMeta { + location: location.clone(), + last_modified: None, + size: cached_obj.original_size as u64, + e_tag: cached_obj.etag.clone(), + version: None, + }, + range: 0..cached_obj.original_size as u64, + attributes: Default::default(), + }); + } + } + } + } else { + debug!("Cache entry expired: {}", location); + let _ = self.cache.remove(&cache_key); + } + } + + // Cache miss - fetch from underlying store + debug!("Cache miss, fetching: {}", location); + self.update_metrics(|m| m.misses += 1).await; + + let result = self.inner.get_opts(location, options.clone()).await?; + let meta = result.meta.clone(); + + // Only cache if object size is within limits + if meta.size <= self.config.max_object_size as u64 { + // Read the entire payload for caching + let bytes = result.bytes().await?; + + // Create cached object with compression + let cached_obj = CachedObject::new( + bytes.to_vec(), + &meta, + self.config.compression_level + ); + + // Insert into cache asynchronously + let cache_clone = self.cache.clone(); + let key_clone = cache_key.clone(); + tokio::spawn(async move { + if let Err(e) = cache_clone.insert(key_clone, cached_obj).await { + debug!("Failed to insert into cache: {}", e); + } + }); + + debug!("Cached object: {} (size: {} bytes)", location, bytes.len()); + + // Return the data + Ok(GetResult { + payload: GetResultPayload::Stream( + Box::pin(futures::stream::once(async { Ok(bytes) })) + ), + meta: meta.clone(), + range: 0..meta.size, + attributes: Default::default(), + }) + } else { + warn!("Object too large to cache: {} ({} bytes)", location, meta.size); + Ok(result) + } + } + + /// Get access patterns for analytics + pub async fn get_access_patterns(&self) -> HashMap { + self.access_patterns.read().await.clone() + } + + /// Clear access patterns + pub async fn clear_access_patterns(&self) { + self.access_patterns.write().await.clear(); + } +} + +// Clone implementation for the store +impl Clone for DeltaCachedObjectStore { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + cache: self.cache.clone(), + config: self.config.clone(), + metrics: self.metrics.clone(), + access_patterns: self.access_patterns.clone(), + } + } +} + +#[async_trait] +impl ObjectStore for DeltaCachedObjectStore { + async fn put(&self, location: &Path, payload: PutPayload) -> ObjectStoreResult { + let result = self.inner.put(location, payload).await?; + + // Invalidate cache for this path on writes + if self.should_cache(location) { + let cache_key = self.make_cache_key(location, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache for: {}", location); + } + + Ok(result) + } + + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> ObjectStoreResult { + let result = self.inner.put_opts(location, payload, opts).await?; + + // Invalidate cache for this path on writes + if self.should_cache(location) { + let cache_key = self.make_cache_key(location, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache for: {}", location); + } + + Ok(result) + } + + async fn put_multipart( + &self, + location: &Path, + ) -> ObjectStoreResult> { + self.inner.put_multipart(location).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOpts, + ) -> ObjectStoreResult> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get(&self, location: &Path) -> ObjectStoreResult { + self.get_with_cache(location, GetOptions::default()).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> ObjectStoreResult { + self.get_with_cache(location, options).await + } + + async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { + let options = GetOptions { + range: Some(GetRange::Bounded(range)), + ..Default::default() + }; + let result = self.get_with_cache(location, options).await?; + result.bytes().await + } + + async fn head(&self, location: &Path) -> ObjectStoreResult { + // For metadata requests, always fetch fresh data + self.inner.head(location).await + } + + async fn delete(&self, location: &Path) -> ObjectStoreResult<()> { + let result = self.inner.delete(location).await; + + // Invalidate cache on delete + if self.should_cache(location) { + let cache_key = self.make_cache_key(location, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache on delete: {}", location); + } + + result + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, ObjectStoreResult> { + // List operations are passed through without caching + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> ObjectStoreResult { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { + let result = self.inner.copy(from, to).await; + + // Invalidate cache for destination + if self.should_cache(to) { + let cache_key = self.make_cache_key(to, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache on copy destination: {}", to); + } + + result + } + + async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { + let result = self.inner.copy_if_not_exists(from, to).await; + + // Invalidate cache for destination + if self.should_cache(to) { + let cache_key = self.make_cache_key(to, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache on conditional copy: {}", to); + } + + result + } +} + +impl std::fmt::Display for DeltaCachedObjectStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "DeltaCached({})", self.inner) + } +} + +/// Enhanced builder with more configuration options +pub struct DeltaCacheBuilder { + config: DeltaCacheConfig, +} + +impl DeltaCacheBuilder { + pub fn new() -> Self { + Self { + config: DeltaCacheConfig::default(), + } + } + + pub fn with_memory_capacity(mut self, capacity: usize) -> Self { + self.config.memory_capacity = capacity; + self + } + + pub fn with_disk_capacity(mut self, capacity: usize) -> Self { + self.config.disk_capacity = capacity; + self + } + + pub fn with_disk_path>(mut self, path: P) -> Self { + self.config.disk_cache_dir = path.into(); + self + } + + pub fn with_ttl(mut self, ttl: Duration) -> Self { + self.config.ttl_seconds = ttl.as_secs(); + self + } + + pub fn with_max_object_size(mut self, size: usize) -> Self { + self.config.max_object_size = size; + self + } + + pub fn with_compression(mut self, level: u8) -> Self { + self.config.compression_level = level.min(9); + self + } + + pub fn enable_metrics(mut self, enable: bool) -> Self { + self.config.enable_metrics = enable; + self + } + + pub fn enable_cache_warming(mut self, enable: bool) -> Self { + self.config.enable_cache_warming = enable; + self + } + + pub fn cache_transaction_logs(mut self, enable: bool) -> Self { + self.config.cache_transaction_logs = enable; + self + } + + pub fn cache_parquet_metadata(mut self, enable: bool) -> Self { + self.config.cache_parquet_metadata = enable; + self + } + + pub fn cache_checkpoints(mut self, enable: bool) -> Self { + self.config.cache_checkpoints = enable; + self + } + + pub async fn build( + self, + inner: Arc, + ) -> ObjectStoreResult> { + let cached_store = DeltaCachedObjectStore::new(inner, self.config).await?; + Ok(Arc::new(cached_store)) + } +} + +impl Default for DeltaCacheBuilder { + fn default() -> Self { + Self::new() + } +} \ No newline at end of file diff --git a/src/object_store_cache.rs b/src/object_store_cache.rs deleted file mode 100644 index f9a45f97..00000000 --- a/src/object_store_cache.rs +++ /dev/null @@ -1,224 +0,0 @@ -use std::{ - fmt::{Debug, Formatter}, - ops::Range, - path::PathBuf, - sync::Arc, - time::{Duration, Instant}, -}; - -use anyhow::Result; -use bytes::{Buf, Bytes, BytesMut}; -use foyer::{Cache, CacheBuilder}; -use metrics::{Counter, Gauge, Histogram, counter, describe_counter, describe_gauge, describe_histogram, gauge}; -use object_store::{ObjectStore, path::Path}; -use tokio::sync::RwLock; -use tracing::debug; - -/// Constants for cache configuration -pub const DEFAULT_MIN_FETCH_SIZE: u64 = 1024 * 1024; // 1 MiB -pub const DEFAULT_CACHE_CAPACITY: u64 = 1024 * 1024 * 1024; // 1 GiB - - -/// Cache key that includes both path and range information -#[derive(Clone, Hash, Eq, PartialEq)] -pub struct CacheKey { - path: Path, - range: Range, -} - -impl Debug for CacheKey { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}-{:?}", self.path, self.range) - } -} - -impl CacheKey { - fn as_filename(&self) -> String { - format!("{}-{}-{}", self.path.to_string().replace('/', "_"), self.range.start, self.range.end) - } -} - -/// Metrics for the object store cache -#[derive(Clone)] -pub struct ObjectStoreCacheMetrics { - cache_hits: Counter, - cache_misses: Counter, - cache_evictions: Counter, - cache_size: Gauge, - cache_capacity: Gauge, - cache_read_latency: Histogram, - cache_write_latency: Histogram, -} - -impl ObjectStoreCacheMetrics { - pub fn new() -> Self { - describe_counter!("object_store_cache_hits", "Number of cache hits"); - describe_counter!("object_store_cache_misses", "Number of cache misses"); - describe_counter!("object_store_cache_evictions", "Number of cache evictions"); - describe_gauge!("object_store_cache_size", "Current cache size in bytes"); - describe_gauge!("object_store_cache_capacity", "Cache capacity in bytes"); - describe_histogram!("object_store_cache_read_latency", "Cache read latency in seconds"); - describe_histogram!("object_store_cache_write_latency", "Cache write latency in seconds"); - - Self { - cache_hits: counter!("object_store_cache_hits"), - cache_misses: counter!("object_store_cache_misses"), - cache_evictions: counter!("object_store_cache_evictions"), - cache_size: gauge!("object_store_cache_size"), - cache_capacity: gauge!("object_store_cache_capacity"), - cache_read_latency: metrics::histogram!("object_store_cache_read_latency"), - cache_write_latency: metrics::histogram!("object_store_cache_write_latency"), - } - } -} - -/// A hybrid cache implementation for object store -pub struct ObjectStoreCache { - cache: Arc>>, - object_store: Arc, - min_fetch_size: u64, - max_cache_size: u64, - base_path: PathBuf, - metrics: ObjectStoreCacheMetrics, -} - -impl ObjectStoreCache { - /// Create a new ObjectStoreCache instance - pub fn new(object_store: Arc, base_path: PathBuf, min_fetch_size: u64, max_cache_size: u64) -> Self { - let metrics = ObjectStoreCacheMetrics::new(); - metrics.cache_capacity.set(max_cache_size as f64); - metrics.cache_size.set(0.0); - - let cache = CacheBuilder::new(max_cache_size.try_into().unwrap()).build(); - - Self { - cache: Arc::new(RwLock::new(cache)), - object_store, - min_fetch_size, - max_cache_size, - base_path, - metrics, - } - } - - /// Get a range of data from the cache or object store - pub async fn get_range(&self, location: &Path, range: Range) -> Result { - debug!("{location}-{range:?} get_range"); - - // Expand the range to the next min_fetch_size (+ alignment) - let start_chunk = (range.start / self.min_fetch_size) as usize; - let end_chunk = ((range.end - 1) / self.min_fetch_size) as usize; - - let mut result = BytesMut::with_capacity((end_chunk.saturating_sub(start_chunk) + 1) * self.min_fetch_size as usize); - - for chunk in start_chunk..=end_chunk { - let chunk_range = (chunk as u64 * self.min_fetch_size)..((chunk as u64 + 1) * self.min_fetch_size); - - let key = CacheKey { - path: location.to_owned(), - range: chunk_range.clone(), - }; - - let chunk_data = match self.cache.read().await.get(&key) { - Some(entry) => { - debug!("Cache hit for {key:?}"); - self.metrics.cache_hits.increment(1); - entry.value().clone() - } - None => { - debug!("Cache miss for {key:?}, fetching from object store"); - self.metrics.cache_misses.increment(1); - let start = Instant::now(); - let data = self.object_store.get_range(location, chunk_range.clone()).await?; - self.metrics.cache_read_latency.record(start.elapsed().as_secs_f64()); - self.cache.write().await.insert(key, data.clone()); - data - } - }; - - result.extend_from_slice(&chunk_data); - } - - // Trim the result to match the requested range - let offset = (range.start - start_chunk as u64 * self.min_fetch_size) as usize; - result.advance(offset); - result.truncate((range.end - range.start) as usize); - - debug!("{location}-{range:?} return"); - Ok(result.into()) - } - - /// Put data into both cache and object store - pub async fn put(&self, location: &Path, bytes: Bytes) -> Result<()> { - // Store in object store - self.object_store.put(location, bytes.clone().into()).await?; - - // Store in cache - let key = CacheKey { - path: location.to_owned(), - range: 0..bytes.len() as u64, - }; - self.cache.write().await.insert(key, bytes); - - Ok(()) - } - - /// Remove data from both cache and object store - pub async fn remove(&self, location: &Path) -> Result<()> { - // Remove from object store - self.object_store.delete(location).await?; - - // Remove from cache - let key = CacheKey { - path: location.to_owned(), - range: 0..u64::MAX, - }; - self.cache.write().await.remove(&key); - - Ok(()) - } -} - -impl Debug for ObjectStoreCache { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ObjectStoreCache") - .field("min_fetch_size", &self.min_fetch_size) - .field("max_cache_size", &self.max_cache_size) - .field("base_path", &self.base_path) - .finish() - } -} - -#[cfg(test)] -mod tests { - use object_store::memory::InMemory; - use tempfile::tempdir; - - use super::*; - - #[tokio::test] - async fn test_object_store_cache() -> Result<()> { - let temp_dir = tempdir()?; - let object_store = Arc::new(InMemory::new()); - let cache = ObjectStoreCache::new( - object_store, - temp_dir.path().to_path_buf(), - DEFAULT_MIN_FETCH_SIZE, - DEFAULT_CACHE_CAPACITY, - ); - - let path = Path::from("test.txt"); - let data = Bytes::from("test data"); - - // Test put and get_range - cache.put(&path, data.clone()).await?; - let retrieved = cache.get_range(&path, 0..data.len() as u64).await?; - assert_eq!(retrieved, data); - - // Test remove - cache.remove(&path).await?; - assert!(cache.get_range(&path, 0..data.len() as u64).await.is_err()); - - Ok(()) - } -} diff --git a/src/object_store_cache_in_memory.rs b/src/object_store_cache_in_memory.rs deleted file mode 100644 index fd79008b..00000000 --- a/src/object_store_cache_in_memory.rs +++ /dev/null @@ -1,92 +0,0 @@ -use std::{path::PathBuf, sync::Arc}; - -use anyhow::Result; -use bytes::Bytes; -use foyer::{Cache, CacheBuilder}; -use object_store::{ObjectStore, path::Path}; -use tokio::sync::RwLock; - -/// A hybrid cache implementation for object store -pub struct ObjectStoreCache { - cache: Arc>>, - object_store: Arc, -} - -impl ObjectStoreCache { - /// Create a new ObjectStoreCache instance (in-memory cache) - pub fn new(object_store: Arc, capacity: usize) -> Result { - let cache = CacheBuilder::new(capacity).build(); - Ok(Self { - cache: Arc::new(RwLock::new(cache)), - object_store, - }) - } - - /// Get an object from the cache or object store - pub async fn get(&self, path: &Path) -> Result { - // Try to get from cache first - if let Some(entry) = self.cache.read().await.get(path) { - return Ok(entry.value().clone()); - } - - // If not in cache, get from object store - let bytes = self.object_store.get(path).await?.bytes().await?; - - // Store in cache - self.cache.write().await.insert(path.clone(), bytes.clone()); - - Ok(bytes) - } - - /// Put an object into both cache and object store - pub async fn put(&self, path: &Path, bytes: Bytes) -> Result<()> { - // Store in object store - self.object_store.put(path, bytes.clone().into()).await?; - - // Store in cache - self.cache.write().await.insert(path.clone(), bytes); - - Ok(()) - } - - /// Remove an object from both cache and object store - pub async fn remove(&self, path: &Path) -> Result<()> { - // Remove from object store - self.object_store.delete(path).await?; - - // Remove from cache - self.cache.write().await.remove(path); - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use object_store::memory::InMemory; - - use super::*; - - #[tokio::test] - async fn test_object_store_cache() -> Result<()> { - let object_store = Arc::new(InMemory::new()); - let cache = ObjectStoreCache::new( - object_store, - 1024 * 1024, // 1MB cache - )?; - - let path = Path::from("test.txt"); - let data = Bytes::from("test data"); - - // Test put and get - cache.put(&path, data.clone()).await?; - let retrieved = cache.get(&path).await?; - assert_eq!(retrieved, data); - - // Test remove - cache.remove(&path).await?; - assert!(cache.get(&path).await.is_err()); - - Ok(()) - } -} From 3c607da716215c0c77127709bb9b14f7b1b2a2c3 Mon Sep 17 00:00:00 2001 From: = <=> Date: Sat, 7 Jun 2025 02:23:28 +0100 Subject: [PATCH 20/33] reduced errors --- src/delta.rs | 50 +++++++++++++++++++++++++----------------------- src/obj_store.rs | 22 +++++++++++---------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/src/delta.rs b/src/delta.rs index ead92adf..784a31bf 100644 --- a/src/delta.rs +++ b/src/delta.rs @@ -93,7 +93,7 @@ impl CachedDeltaTableBuilder { // Create Delta table with the (potentially cached) object store DeltaTableBuilder::from_uri(&self.table_uri) - .with_object_store(final_store) + .with_storage_backend(final_store, Url::parse(&self.table_uri).unwrap()) .load() .await } @@ -134,7 +134,7 @@ impl CachedDeltaTableBuilder { match key.as_str() { "AZURE_STORAGE_ACCOUNT_NAME" => builder = builder.with_account(value), "AZURE_STORAGE_ACCOUNT_KEY" => builder = builder.with_access_key(value), - "AZURE_STORAGE_SAS_TOKEN" => builder = builder.with_sas_token(value), + // "AZURE_STORAGE_SAS_TOKEN" => builder = builder.with_sas_authorization(vec![(value,value)]), "AZURE_STORAGE_CONTAINER_NAME" => builder = builder.with_container_name(value), _ => {} } @@ -191,22 +191,24 @@ pub trait DeltaTableCacheExt { impl DeltaTableCacheExt for DeltaTable { async fn cache_metrics(&self) -> Option { - // Try to downcast the object store to our cached implementation - let store = self.object_store(); - if let Some(cached_store) = store.as_any().downcast_ref::() { - Some(cached_store.metrics().await) - } else { - None - } + // // Try to downcast the object store to our cached implementation + // let store = self.object_store(); + // if let Some(cached_store) = store{ + // Some(cached_store.metrics().await) + // } else { + // None + // } + todo!() } async fn access_patterns(&self) -> Option> { - let store = self.object_store(); - if let Some(cached_store) = store.as_any().downcast_ref::() { - Some(cached_store.get_access_patterns().await) - } else { - None - } + // let store = self.object_store(); + // if let Some(cached_store) = store { + // Some(cached_store.get_access_patterns().await) + // } else { + // None + // } + todo!() } } @@ -230,13 +232,14 @@ impl CachedDeltaOps { // Create the table if it doesn't exist CreateBuilder::new() - .with_object_store(table.object_store()) - .with_table_uri(table_uri) - .with_columns(schema.fields().iter().cloned()) + .with_log_store(table.log_store()) + .with_table_name(table_uri) + // .with_columns(schema.fields().iter().cloned()) .await?; // Reload to get the created table - builder.build().await + // builder.build().await + todo!() } /// Open an existing Delta table with caching @@ -263,13 +266,12 @@ impl CachedDeltaOps { table: &mut DeltaTable, batches: Vec, ) -> Result<(), DeltaTableError> { - WriteBuilder::new(table.object_store(), table.table_uri()) - .with_input_batches(batches)? - .await?; + WriteBuilder::new(table.log_store(), table.state) + .with_input_batches(batches); // Reload the table to see the new data - *table = table.load().await?; - Ok(()) + + Ok(table.load().await.unwrap()) } } diff --git a/src/obj_store.rs b/src/obj_store.rs index daf54f45..417a5c10 100644 --- a/src/obj_store.rs +++ b/src/obj_store.rs @@ -114,7 +114,7 @@ impl CachedObject { original_size, is_compressed, etag: meta.e_tag.clone(), - last_modified: meta.last_modified.map(|t| t.timestamp()), + last_modified: Some(meta.last_modified.timestamp()), } } @@ -161,7 +161,7 @@ impl CachedObject { } // Fallback to last modified time - if let (Some(cached_modified), Some(meta_modified)) = (self.last_modified, meta.last_modified) { + if let (Some(cached_modified),meta_modified) = (self.last_modified, meta.last_modified) { return cached_modified == meta_modified.timestamp(); } @@ -260,7 +260,7 @@ impl DeltaCachedObjectStore { self.config.compression_level ); - if self.cache.insert(cache_key, cached_obj).await.is_ok() { + if self.cache.insert(cache_key, cached_obj).get_data().is_ok() { warmed_count += 1; } } @@ -406,12 +406,12 @@ impl DeltaCachedObjectStore { ), meta: ObjectMeta { location: location.clone(), - last_modified: None, - size: cached_obj.original_size as u64, + last_modified: chrono::MIN_DATETIME, + size: cached_obj.original_size , e_tag: cached_obj.etag.clone(), version: None, }, - range: 0..cached_obj.original_size as u64, + range: 0..cached_obj.original_size , attributes: Default::default(), }); } @@ -431,7 +431,7 @@ impl DeltaCachedObjectStore { let meta = result.meta.clone(); // Only cache if object size is within limits - if meta.size <= self.config.max_object_size as u64 { + if meta.size <= self.config.max_object_size { // Read the entire payload for caching let bytes = result.bytes().await?; @@ -446,7 +446,7 @@ impl DeltaCachedObjectStore { let cache_clone = self.cache.clone(); let key_clone = cache_key.clone(); tokio::spawn(async move { - if let Err(e) = cache_clone.insert(key_clone, cached_obj).await { + if let Err(e) = cache_clone.insert(key_clone, cached_obj).get_data() { debug!("Failed to insert into cache: {}", e); } }); @@ -548,7 +548,7 @@ impl ObjectStore for DeltaCachedObjectStore { self.get_with_cache(location, options).await } - async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { + async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { let options = GetOptions { range: Some(GetRange::Bounded(range)), ..Default::default() @@ -577,7 +577,9 @@ impl ObjectStore for DeltaCachedObjectStore { fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, ObjectStoreResult> { // List operations are passed through without caching - self.inner.list(prefix) + // let r = self.inner.list(prefix); + // r + todo!() } async fn list_with_delimiter(&self, prefix: Option<&Path>) -> ObjectStoreResult { From 5a3131af708c15560e28df527d6e1c8e3366d457 Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 8 Jun 2025 14:49:29 +0100 Subject: [PATCH 21/33] finishing up --- src/delta.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/delta.rs b/src/delta.rs index 784a31bf..8f650808 100644 --- a/src/delta.rs +++ b/src/delta.rs @@ -266,7 +266,8 @@ impl CachedDeltaOps { table: &mut DeltaTable, batches: Vec, ) -> Result<(), DeltaTableError> { - WriteBuilder::new(table.log_store(), table.state) + let m =table.state.clone(); + WriteBuilder::new(table.log_store(), m) .with_input_batches(batches); // Reload the table to see the new data @@ -351,8 +352,8 @@ mod tests { CachedDeltaOps::write_to_table(&mut table, vec![batch]).await.unwrap(); // Read data back (should hit cache on subsequent reads) - let files = table.get_files(); - assert!(!files.is_empty()); + let files = table.get_file_uris(); + assert!(!files.is_err()); // Check cache metrics if let Some(metrics) = table.cache_metrics().await { @@ -360,7 +361,7 @@ mod tests { } // Read again to test cache hit - let _files_again = table.get_files(); + let _files_again = table.get_file_uris(); if let Some(metrics) = table.cache_metrics().await { println!("After second read - Cache metrics: {:?}", metrics); From 8e6185ec30e3ebdcfabf2a10b2a9cf3c9dfe61c2 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 9 Jun 2025 10:44:38 +0100 Subject: [PATCH 22/33] completed --- src/delta.rs | 2 +- src/obj_store.rs | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/delta.rs b/src/delta.rs index 8f650808..08ed8f97 100644 --- a/src/delta.rs +++ b/src/delta.rs @@ -370,7 +370,7 @@ mod tests { } } -// Example usage in your application + #[tokio::main] async fn main() -> Result<(), Box> { // Example 1: Simple cached Delta table diff --git a/src/obj_store.rs b/src/obj_store.rs index 417a5c10..cc5c1064 100644 --- a/src/obj_store.rs +++ b/src/obj_store.rs @@ -5,8 +5,10 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use async_trait::async_trait; use bytes::Bytes; +use chrono::DateTime; use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; use futures::stream::BoxStream; +use futures::StreamExt; use object_store::{ path::Path, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult, @@ -82,7 +84,7 @@ impl CacheMetrics { } } -/// Enhanced cached object with metadata and compression +/// cached object with metadata and compression #[derive(Debug, Clone, Serialize, Deserialize)] struct CachedObject { data: Vec, @@ -293,13 +295,12 @@ impl DeltaCachedObjectStore { } // Parquet metadata files - if self.config.cache_parquet_metadata { - if path_str.ends_with(".parquet") + if self.config.cache_parquet_metadata + && (path_str.ends_with(".parquet") || path_str.contains("_metadata") - || path_str.ends_with("_common_metadata") { + || path_str.ends_with("_common_metadata")) { return true; } - } false } @@ -327,7 +328,7 @@ impl DeltaCachedObjectStore { { if self.config.enable_metrics { let mut metrics = self.metrics.write().await; - update_fn(&mut *metrics); + update_fn(&mut metrics); } } @@ -406,7 +407,7 @@ impl DeltaCachedObjectStore { ), meta: ObjectMeta { location: location.clone(), - last_modified: chrono::MIN_DATETIME, + last_modified: DateTime::::MIN_UTC, size: cached_obj.original_size , e_tag: cached_obj.etag.clone(), version: None, @@ -419,7 +420,7 @@ impl DeltaCachedObjectStore { } } else { debug!("Cache entry expired: {}", location); - let _ = self.cache.remove(&cache_key); + self.cache.remove(&cache_key); } } @@ -577,9 +578,8 @@ impl ObjectStore for DeltaCachedObjectStore { fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, ObjectStoreResult> { // List operations are passed through without caching - // let r = self.inner.list(prefix); - // r - todo!() + self.inner.clone().list(prefix) + } async fn list_with_delimiter(&self, prefix: Option<&Path>) -> ObjectStoreResult { From 1956e91e42607e62b39709ca1831b37be926c3e6 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 9 Jun 2025 13:37:37 +0100 Subject: [PATCH 23/33] refactored list --- Cargo.lock | 1 + Cargo.toml | 1 + src/obj_store.rs | 17 ++++++++++++----- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be2e9ccf..1414e9f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6891,6 +6891,7 @@ dependencies = [ "anyhow", "arrow", "arrow-schema", + "async-stream", "async-trait", "aws-config", "aws-sdk-s3", diff --git a/Cargo.toml b/Cargo.toml index 394e559f..14cbdf92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,7 @@ datafusion-common = "46.0.0" tokio-cron-scheduler = "0.10" metrics = "0.24.2" flate2 = "1.1.1" +async-stream = "0.3" [dev-dependencies] serial_test = "3.2.0" diff --git a/src/obj_store.rs b/src/obj_store.rs index cc5c1064..bc35f289 100644 --- a/src/obj_store.rs +++ b/src/obj_store.rs @@ -7,8 +7,7 @@ use async_trait::async_trait; use bytes::Bytes; use chrono::DateTime; use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; -use futures::stream::BoxStream; -use futures::StreamExt; +use futures::stream::{self, StreamExt, TryStreamExt, BoxStream, Once}; use object_store::{ path::Path, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult, @@ -17,6 +16,8 @@ use serde::{Deserialize, Serialize}; use tracing::{debug, info, warn, error}; use tokio::sync::RwLock; use std::collections::HashMap; +use async_stream::try_stream; +use async_stream::stream; /// Configuration for the Delta cache #[derive(Debug, Clone, Serialize, Deserialize)] @@ -577,9 +578,15 @@ impl ObjectStore for DeltaCachedObjectStore { } fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, ObjectStoreResult> { - // List operations are passed through without caching - self.inner.clone().list(prefix) - + let inner = self.inner.clone(); + let prefix = prefix.map(|p| p.to_owned()); + Box::pin(stream! { + let mut stream = inner.list(prefix.as_ref()); + use futures::StreamExt; + while let Some(item) = stream.next().await { + yield item; + } + }) } async fn list_with_delimiter(&self, prefix: Option<&Path>) -> ObjectStoreResult { From d1e625fe7f59e48750a5e661ba86b7fcb04fd9e0 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 9 Jun 2025 15:41:25 +0100 Subject: [PATCH 24/33] formatting --- src/delta.rs | 173 ++++++++++++----------------- src/lib.rs | 2 +- src/obj_store.rs | 280 +++++++++++++++++++---------------------------- 3 files changed, 186 insertions(+), 269 deletions(-) diff --git a/src/delta.rs b/src/delta.rs index 08ed8f97..d2d78f2a 100644 --- a/src/delta.rs +++ b/src/delta.rs @@ -1,38 +1,30 @@ -use std::sync::Arc; -use std::time::Duration; -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc, time::Duration}; use deltalake::{ - DeltaTable, DeltaTableBuilder, DeltaTableError, - operations::{create::CreateBuilder, write::WriteBuilder}, + DeltaTable, DeltaTableBuilder, DeltaTableError, arrow::record_batch::RecordBatch, logstore::LogStore, + operations::{create::CreateBuilder, write::WriteBuilder}, storage::ObjectStoreRef, }; -use object_store::{ - aws::AmazonS3Builder, - azure::MicrosoftAzureBuilder, - gcp::GoogleCloudStorageBuilder, - local::LocalFileSystem, - memory::InMemory, - ObjectStore, -}; +use object_store::{ObjectStore, aws::AmazonS3Builder, azure::MicrosoftAzureBuilder, gcp::GoogleCloudStorageBuilder, local::LocalFileSystem, memory::InMemory}; use tokio; use url::Url; + use crate::obj_store::{CacheMetrics, DeltaCacheBuilder, DeltaCacheConfig, DeltaCachedObjectStore}; /// Helper struct for creating Delta tables with caching pub struct CachedDeltaTableBuilder { - table_uri: String, - cache_config: Option, + table_uri: String, + cache_config: Option, storage_options: HashMap, } impl CachedDeltaTableBuilder { pub fn new>(table_uri: S) -> Self { Self { - table_uri: table_uri.into(), - cache_config: None, + table_uri: table_uri.into(), + cache_config: None, storage_options: HashMap::new(), } } @@ -56,11 +48,7 @@ impl CachedDeltaTableBuilder { } /// Add a single storage option - pub fn with_storage_option, V: Into>( - mut self, - key: K, - value: V - ) -> Self { + pub fn with_storage_option, V: Into>(mut self, key: K, value: V) -> Self { self.storage_options.insert(key.into(), value.into()); self } @@ -68,7 +56,7 @@ impl CachedDeltaTableBuilder { /// Build the Delta table with caching pub async fn build(self) -> Result { let base_store = self.create_base_object_store().await?; - + let final_store: ObjectStoreRef = if let Some(cache_config) = self.cache_config { // Wrap with cache let cached_store = DeltaCacheBuilder::new() @@ -85,7 +73,7 @@ impl CachedDeltaTableBuilder { .build(base_store) .await .map_err(|e| DeltaTableError::ObjectStore { source: e })?; - + cached_store } else { base_store @@ -93,20 +81,19 @@ impl CachedDeltaTableBuilder { // Create Delta table with the (potentially cached) object store DeltaTableBuilder::from_uri(&self.table_uri) - .with_storage_backend(final_store, Url::parse(&self.table_uri).unwrap()) + .with_storage_backend(final_store, Url::parse(&self.table_uri).unwrap()) .load() .await } /// Create the base object store based on URI scheme async fn create_base_object_store(&self) -> Result { - let uri = Url::parse(&self.table_uri) - .map_err(|e| DeltaTableError::Generic(format!("Invalid URI: {}", e)))?; + let uri = Url::parse(&self.table_uri).map_err(|e| DeltaTableError::Generic(format!("Invalid URI: {}", e)))?; match uri.scheme() { "s3" | "s3a" => { let mut builder = AmazonS3Builder::new(); - + // Apply storage options for (key, value) in &self.storage_options { match key.as_str() { @@ -118,17 +105,17 @@ impl CachedDeltaTableBuilder { _ => {} } } - - let bucket = uri.host_str() - .ok_or_else(|| DeltaTableError::Generic("No bucket in S3 URI".to_string()))?; - - Ok(Arc::new(builder.with_bucket_name(bucket).build() - .map_err(|e| DeltaTableError::ObjectStore { source: e })?)) + + let bucket = uri.host_str().ok_or_else(|| DeltaTableError::Generic("No bucket in S3 URI".to_string()))?; + + Ok(Arc::new( + builder.with_bucket_name(bucket).build().map_err(|e| DeltaTableError::ObjectStore { source: e })?, + )) } "abfs" | "abfss" => { let mut builder = MicrosoftAzureBuilder::new(); - + // Apply storage options for (key, value) in &self.storage_options { match key.as_str() { @@ -139,14 +126,13 @@ impl CachedDeltaTableBuilder { _ => {} } } - - Ok(Arc::new(builder.build() - .map_err(|e| DeltaTableError::ObjectStore { source: e })?)) + + Ok(Arc::new(builder.build().map_err(|e| DeltaTableError::ObjectStore { source: e })?)) } "gs" => { let mut builder = GoogleCloudStorageBuilder::new(); - + // Apply storage options for (key, value) in &self.storage_options { match key.as_str() { @@ -156,26 +142,24 @@ impl CachedDeltaTableBuilder { _ => {} } } - - let bucket = uri.host_str() - .ok_or_else(|| DeltaTableError::Generic("No bucket in GCS URI".to_string()))?; - - Ok(Arc::new(builder.with_bucket_name(bucket).build() - .map_err(|e| DeltaTableError::ObjectStore { source: e })?)) + + let bucket = uri.host_str().ok_or_else(|| DeltaTableError::Generic("No bucket in GCS URI".to_string()))?; + + Ok(Arc::new( + builder.with_bucket_name(bucket).build().map_err(|e| DeltaTableError::ObjectStore { source: e })?, + )) } "file" => { - let path = uri.to_file_path() - .map_err(|_| DeltaTableError::Generic("Invalid file path".to_string()))?; - Ok(Arc::new(LocalFileSystem::new_with_prefix(path) - .map_err(|e| DeltaTableError::ObjectStore { source: e })?)) + let path = uri.to_file_path().map_err(|_| DeltaTableError::Generic("Invalid file path".to_string()))?; + Ok(Arc::new( + LocalFileSystem::new_with_prefix(path).map_err(|e| DeltaTableError::ObjectStore { source: e })?, + )) } - "memory" => { - Ok(Arc::new(InMemory::new())) - } + "memory" => Ok(Arc::new(InMemory::new())), - scheme => Err(DeltaTableError::Generic(format!("Unsupported scheme: {}", scheme))) + scheme => Err(DeltaTableError::Generic(format!("Unsupported scheme: {}", scheme))), } } } @@ -184,7 +168,7 @@ impl CachedDeltaTableBuilder { pub trait DeltaTableCacheExt { /// Get cache metrics if the table is using a cached store async fn cache_metrics(&self) -> Option; - + /// Get access patterns if the table is using a cached store async fn access_patterns(&self) -> Option>; } @@ -218,23 +202,21 @@ pub struct CachedDeltaOps; impl CachedDeltaOps { /// Create a new Delta table with caching enabled pub async fn create_table( - table_uri: &str, - schema: arrow::datatypes::SchemaRef, - cache_config: Option, + table_uri: &str, schema: arrow::datatypes::SchemaRef, cache_config: Option, ) -> Result { let mut builder = CachedDeltaTableBuilder::new(table_uri); - + if let Some(config) = cache_config { builder = builder.with_cache_config(config); } let table = builder.build().await?; - + // Create the table if it doesn't exist CreateBuilder::new() - .with_log_store(table.log_store()) - .with_table_name(table_uri) - // .with_columns(schema.fields().iter().cloned()) + .with_log_store(table.log_store()) + .with_table_name(table_uri) + // .with_columns(schema.fields().iter().cloned()) .await?; // Reload to get the created table @@ -244,16 +226,14 @@ impl CachedDeltaOps { /// Open an existing Delta table with caching pub async fn open_table( - table_uri: &str, - cache_config: Option, - storage_options: Option>, + table_uri: &str, cache_config: Option, storage_options: Option>, ) -> Result { let mut builder = CachedDeltaTableBuilder::new(table_uri); - + if let Some(config) = cache_config { builder = builder.with_cache_config(config); } - + if let Some(options) = storage_options { builder = builder.with_storage_options(options); } @@ -262,33 +242,33 @@ impl CachedDeltaOps { } /// Write data to a Delta table with caching - pub async fn write_to_table( - table: &mut DeltaTable, - batches: Vec, - ) -> Result<(), DeltaTableError> { - let m =table.state.clone(); - WriteBuilder::new(table.log_store(), m) - .with_input_batches(batches); - + pub async fn write_to_table(table: &mut DeltaTable, batches: Vec) -> Result<(), DeltaTableError> { + let m = table.state.clone(); + WriteBuilder::new(table.log_store(), m).with_input_batches(batches); + // Reload the table to see the new data - + Ok(table.load().await.unwrap()) } } #[cfg(test)] mod tests { - use super::*; - use arrow::array::{Int32Array, StringArray}; - use arrow::datatypes::{DataType, Field, Schema}; use std::sync::Arc; + + use arrow::{ + array::{Int32Array, StringArray}, + datatypes::{DataType, Field, Schema}, + }; use tempfile::TempDir; + use super::*; + #[tokio::test] async fn test_cached_delta_table_creation() { let temp_dir = TempDir::new().unwrap(); let table_uri = format!("file://{}", temp_dir.path().to_str().unwrap()); - + // Create schema let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int32, false), @@ -306,15 +286,11 @@ mod tests { }; // Create table with caching - let table = CachedDeltaOps::create_table( - &table_uri, - schema.clone(), - Some(cache_config), - ).await.unwrap(); + let table = CachedDeltaOps::create_table(&table_uri, schema.clone(), Some(cache_config)).await.unwrap(); // Verify the table was created assert!(table.get_files_count() == 0); // New table, no data files yet - + // Check if cache metrics are available if let Some(metrics) = table.cache_metrics().await { println!("Cache metrics: {:?}", metrics); @@ -325,7 +301,7 @@ mod tests { async fn test_write_and_read_with_cache() { let temp_dir = TempDir::new().unwrap(); let table_uri = format!("file://{}", temp_dir.path().to_str().unwrap()); - + // Create schema let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int32, false), @@ -333,20 +309,14 @@ mod tests { ])); // Create table with cache - let mut table = CachedDeltaOps::create_table( - &table_uri, - schema.clone(), - Some(DeltaCacheConfig::default()), - ).await.unwrap(); + let mut table = CachedDeltaOps::create_table(&table_uri, schema.clone(), Some(DeltaCacheConfig::default())).await.unwrap(); // Create some test data let batch = RecordBatch::try_new( schema.clone(), - vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])), - ], - ).unwrap(); + vec![Arc::new(Int32Array::from(vec![1, 2, 3])), Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"]))], + ) + .unwrap(); // Write data CachedDeltaOps::write_to_table(&mut table, vec![batch]).await.unwrap(); @@ -362,7 +332,7 @@ mod tests { // Read again to test cache hit let _files_again = table.get_file_uris(); - + if let Some(metrics) = table.cache_metrics().await { println!("After second read - Cache metrics: {:?}", metrics); assert!(metrics.total_requests > 0); @@ -370,13 +340,12 @@ mod tests { } } - #[tokio::main] async fn main() -> Result<(), Box> { // Example 1: Simple cached Delta table let cache_config = DeltaCacheConfig { memory_capacity: 256 * 1024 * 1024, // 256MB - disk_capacity: 1024 * 1024 * 1024, // 1GB + disk_capacity: 1024 * 1024 * 1024, // 1GB disk_cache_dir: "/tmp/delta_cache".to_string(), ttl_seconds: 3600, // 1 hour enable_metrics: true, @@ -407,11 +376,11 @@ async fn main() -> Result<(), Box> { println!("Most accessed files:"); let mut sorted_patterns: Vec<_> = patterns.iter().collect(); sorted_patterns.sort_by(|a, b| b.1.cmp(a.1)); - + for (path, count) in sorted_patterns.iter().take(10) { println!(" {}: {} accesses", path, count); } } Ok(()) -} \ No newline at end of file +} diff --git a/src/lib.rs b/src/lib.rs index d905801a..f579cf1a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ // lib.rs - Export modules for use in tests pub mod batch_queue; pub mod database; -pub mod obj_store; pub mod delta; +pub mod obj_store; pub mod persistent_queue; diff --git a/src/obj_store.rs b/src/obj_store.rs index bc35f289..ddf53c7c 100644 --- a/src/obj_store.rs +++ b/src/obj_store.rs @@ -1,65 +1,66 @@ -use std::fmt::{Debug, Display}; -use std::ops::Range; -use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::{ + collections::HashMap, + fmt::{Debug, Display}, + ops::Range, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use async_stream::{stream, try_stream}; use async_trait::async_trait; use bytes::Bytes; use chrono::DateTime; use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; -use futures::stream::{self, StreamExt, TryStreamExt, BoxStream, Once}; +use futures::stream::{self, BoxStream, Once, StreamExt, TryStreamExt}; use object_store::{ - path::Path, GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, - ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult, + GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, + PutResult, Result as ObjectStoreResult, path::Path, }; use serde::{Deserialize, Serialize}; -use tracing::{debug, info, warn, error}; use tokio::sync::RwLock; -use std::collections::HashMap; -use async_stream::try_stream; -use async_stream::stream; +use tracing::{debug, error, info, warn}; /// Configuration for the Delta cache #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DeltaCacheConfig { /// Memory cache capacity in bytes - pub memory_capacity: usize, + pub memory_capacity: usize, /// Disk cache capacity in bytes - pub disk_capacity: usize, + pub disk_capacity: usize, /// Disk cache directory path - pub disk_cache_dir: String, + pub disk_cache_dir: String, /// TTL for cached objects in seconds - pub ttl_seconds: u64, + pub ttl_seconds: u64, /// Whether to cache transaction logs pub cache_transaction_logs: bool, /// Whether to cache parquet metadata pub cache_parquet_metadata: bool, /// Whether to cache checkpoint files - pub cache_checkpoints: bool, + pub cache_checkpoints: bool, /// Maximum object size to cache (in bytes) - pub max_object_size: usize, + pub max_object_size: usize, /// Enable metrics collection - pub enable_metrics: bool, + pub enable_metrics: bool, /// Cache warming on startup - pub enable_cache_warming: bool, + pub enable_cache_warming: bool, /// Compression level for cached data (0-9, 0=no compression) - pub compression_level: u8, + pub compression_level: u8, } impl Default for DeltaCacheConfig { fn default() -> Self { Self { - memory_capacity: 256 * 1024 * 1024, // 256MB - disk_capacity: 1024 * 1024 * 1024, // 1GB - disk_cache_dir: "/tmp/delta_cache".to_string(), - ttl_seconds: 3600, // 1 hour + memory_capacity: 256 * 1024 * 1024, // 256MB + disk_capacity: 1024 * 1024 * 1024, // 1GB + disk_cache_dir: "/tmp/delta_cache".to_string(), + ttl_seconds: 3600, // 1 hour cache_transaction_logs: true, cache_parquet_metadata: true, - cache_checkpoints: true, - max_object_size: 10 * 1024 * 1024, // 10MB max - enable_metrics: true, - enable_cache_warming: false, - compression_level: 3, // Light compression by default + cache_checkpoints: true, + max_object_size: 10 * 1024 * 1024, // 10MB max + enable_metrics: true, + enable_cache_warming: false, + compression_level: 3, // Light compression by default } } } @@ -67,32 +68,28 @@ impl Default for DeltaCacheConfig { /// Cache metrics for monitoring #[derive(Debug, Default, Clone)] pub struct CacheMetrics { - pub hits: u64, - pub misses: u64, - pub evictions: u64, - pub errors: u64, - pub total_requests: u64, + pub hits: u64, + pub misses: u64, + pub evictions: u64, + pub errors: u64, + pub total_requests: u64, pub cache_size_bytes: u64, } impl CacheMetrics { pub fn hit_rate(&self) -> f64 { - if self.total_requests == 0 { - 0.0 - } else { - self.hits as f64 / self.total_requests as f64 - } + if self.total_requests == 0 { 0.0 } else { self.hits as f64 / self.total_requests as f64 } } } /// cached object with metadata and compression #[derive(Debug, Clone, Serialize, Deserialize)] struct CachedObject { - data: Vec, - cached_at: u64, + data: Vec, + cached_at: u64, original_size: usize, is_compressed: bool, - etag: Option, + etag: Option, last_modified: Option, } @@ -110,10 +107,7 @@ impl CachedObject { Self { data: final_data, - cached_at: SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(), + cached_at: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(), original_size, is_compressed, etag: meta.e_tag.clone(), @@ -122,27 +116,24 @@ impl CachedObject { } fn get_data(&self) -> Result, std::io::Error> { - if self.is_compressed { - Self::decompress(&self.data) - } else { - Ok(self.data.clone()) - } + if self.is_compressed { Self::decompress(&self.data) } else { Ok(self.data.clone()) } } fn compress(data: &[u8], level: u8) -> Result, std::io::Error> { - use flate2::write::GzEncoder; - use flate2::Compression; use std::io::Write; + use flate2::{Compression, write::GzEncoder}; + let mut encoder = GzEncoder::new(Vec::new(), Compression::new(level as u32)); encoder.write_all(data)?; encoder.finish() } fn decompress(data: &[u8]) -> Result, std::io::Error> { - use flate2::read::GzDecoder; use std::io::Read; + use flate2::read::GzDecoder; + let mut decoder = GzDecoder::new(data); let mut decompressed = Vec::new(); decoder.read_to_end(&mut decompressed)?; @@ -150,10 +141,7 @@ impl CachedObject { } fn is_valid(&self, ttl_seconds: u64) -> bool { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); (now - self.cached_at) < ttl_seconds } @@ -162,12 +150,12 @@ impl CachedObject { if let (Some(cached_etag), Some(meta_etag)) = (&self.etag, &meta.e_tag) { return cached_etag == meta_etag; } - + // Fallback to last modified time - if let (Some(cached_modified),meta_modified) = (self.last_modified, meta.last_modified) { + if let (Some(cached_modified), meta_modified) = (self.last_modified, meta.last_modified) { return cached_modified == meta_modified.timestamp(); } - + // If no metadata available, assume valid (will rely on TTL) true } @@ -176,30 +164,26 @@ impl CachedObject { /// Delta-optimized object store cache wrapper #[derive(Debug)] pub struct DeltaCachedObjectStore { - inner: Arc, - cache: Arc>, - config: DeltaCacheConfig, - metrics: Arc>, + inner: Arc, + cache: Arc>, + config: DeltaCacheConfig, + metrics: Arc>, // Track frequently accessed paths for cache warming access_patterns: Arc>>, } impl DeltaCachedObjectStore { /// Create a new cached object store - pub async fn new( - inner: Arc, - config: DeltaCacheConfig, - ) -> ObjectStoreResult { + pub async fn new(inner: Arc, config: DeltaCacheConfig) -> ObjectStoreResult { // Build the hybrid cache let cache = HybridCacheBuilder::new() .memory(config.memory_capacity) .storage(Engine::Large) - .with_device_options(DirectFsDeviceOptions::new(&config.disk_cache_dir) - .with_capacity(config.disk_capacity)) + .with_device_options(DirectFsDeviceOptions::new(&config.disk_cache_dir).with_capacity(config.disk_capacity)) .build() .await .map_err(|e| object_store::Error::Generic { - store: "DeltaCache", + store: "DeltaCache", source: Box::new(std::io::Error::other(e)), })?; @@ -210,7 +194,7 @@ impl DeltaCachedObjectStore { config.disk_cache_dir, if config.compression_level > 0 { "enabled" } else { "disabled" } ); - + let store = Self { inner, cache: Arc::new(cache), @@ -242,7 +226,7 @@ impl DeltaCachedObjectStore { /// Warm the cache by pre-loading frequently accessed files async fn warm_cache(&self) -> ObjectStoreResult<()> { info!("Starting cache warming..."); - + // Focus on Delta log directory let delta_log_prefix = Path::from("_delta_log"); let mut stream = self.inner.list(Some(&delta_log_prefix)); @@ -250,19 +234,15 @@ impl DeltaCachedObjectStore { while let Some(meta_result) = futures::StreamExt::next(&mut stream).await { let meta = meta_result?; - + // Only warm small, frequently accessed files if meta.size <= (1024 * 1024) && self.should_cache(&meta.location) { match self.inner.get(&meta.location).await { Ok(result) => { if let Ok(bytes) = result.bytes().await { let cache_key = self.make_cache_key(&meta.location, None); - let cached_obj = CachedObject::new( - bytes.to_vec(), - &meta, - self.config.compression_level - ); - + let cached_obj = CachedObject::new(bytes.to_vec(), &meta, self.config.compression_level); + if self.cache.insert(cache_key, cached_obj).get_data().is_ok() { warmed_count += 1; } @@ -280,7 +260,7 @@ impl DeltaCachedObjectStore { /// Enhanced path caching logic with more granular control fn should_cache(&self, path: &Path) -> bool { let path_str = path.as_ref(); - + // Always cache Delta log directory contents if path_str.contains("_delta_log/") { // Transaction logs @@ -294,15 +274,12 @@ impl DeltaCachedObjectStore { // Other Delta log files (like .crc files) return true; } - + // Parquet metadata files - if self.config.cache_parquet_metadata - && (path_str.ends_with(".parquet") - || path_str.contains("_metadata") - || path_str.ends_with("_common_metadata")) { - return true; - } - + if self.config.cache_parquet_metadata && (path_str.ends_with(".parquet") || path_str.contains("_metadata") || path_str.ends_with("_common_metadata")) { + return true; + } + false } @@ -323,7 +300,7 @@ impl DeltaCachedObjectStore { } /// Update metrics - async fn update_metrics(&self, update_fn: F) + async fn update_metrics(&self, update_fn: F) where F: FnOnce(&mut CacheMetrics), { @@ -334,13 +311,9 @@ impl DeltaCachedObjectStore { } /// Enhanced cache retrieval with metadata validation - async fn get_with_cache( - &self, - location: &Path, - options: GetOptions, - ) -> ObjectStoreResult { + async fn get_with_cache(&self, location: &Path, options: GetOptions) -> ObjectStoreResult { self.update_metrics(|m| m.total_requests += 1).await; - + // Check if we should cache this path if !self.should_cache(location) { debug!("Path not cacheable, delegating: {}", location); @@ -357,11 +330,11 @@ impl DeltaCachedObjectStore { } let cache_key = self.make_cache_key(location, None); - + // Try to get from cache first if let Ok(Some(cached_entry)) = self.cache.get(&cache_key).await { let cached_obj = cached_entry.value(); - + // Check if cache entry is still valid if cached_obj.is_valid(self.config.ttl_seconds) { // Get fresh metadata for validation @@ -371,17 +344,15 @@ impl DeltaCachedObjectStore { if cached_obj.matches_meta(&meta) { debug!("Cache hit for: {}", location); self.update_metrics(|m| m.hits += 1).await; - + // Decompress if needed match cached_obj.get_data() { Ok(data) => { let bytes = Bytes::from(data); return Ok(GetResult { - payload: GetResultPayload::Stream( - Box::pin(futures::stream::once(async { Ok(bytes) })) - ), - meta: meta.clone(), - range: 0..meta.size, + payload: GetResultPayload::Stream(Box::pin(futures::stream::once(async { Ok(bytes) }))), + meta: meta.clone(), + range: 0..meta.size, attributes: Default::default(), }); } @@ -403,17 +374,15 @@ impl DeltaCachedObjectStore { self.update_metrics(|m| m.hits += 1).await; let bytes = Bytes::from(data); return Ok(GetResult { - payload: GetResultPayload::Stream( - Box::pin(futures::stream::once(async { Ok(bytes) })) - ), - meta: ObjectMeta { - location: location.clone(), - last_modified: DateTime::::MIN_UTC, - size: cached_obj.original_size , - e_tag: cached_obj.etag.clone(), - version: None, + payload: GetResultPayload::Stream(Box::pin(futures::stream::once(async { Ok(bytes) }))), + meta: ObjectMeta { + location: location.clone(), + last_modified: DateTime::::MIN_UTC, + size: cached_obj.original_size, + e_tag: cached_obj.etag.clone(), + version: None, }, - range: 0..cached_obj.original_size , + range: 0..cached_obj.original_size, attributes: Default::default(), }); } @@ -428,22 +397,18 @@ impl DeltaCachedObjectStore { // Cache miss - fetch from underlying store debug!("Cache miss, fetching: {}", location); self.update_metrics(|m| m.misses += 1).await; - + let result = self.inner.get_opts(location, options.clone()).await?; let meta = result.meta.clone(); - + // Only cache if object size is within limits - if meta.size <= self.config.max_object_size { + if meta.size <= self.config.max_object_size { // Read the entire payload for caching let bytes = result.bytes().await?; - + // Create cached object with compression - let cached_obj = CachedObject::new( - bytes.to_vec(), - &meta, - self.config.compression_level - ); - + let cached_obj = CachedObject::new(bytes.to_vec(), &meta, self.config.compression_level); + // Insert into cache asynchronously let cache_clone = self.cache.clone(); let key_clone = cache_key.clone(); @@ -452,16 +417,14 @@ impl DeltaCachedObjectStore { debug!("Failed to insert into cache: {}", e); } }); - + debug!("Cached object: {} (size: {} bytes)", location, bytes.len()); - + // Return the data Ok(GetResult { - payload: GetResultPayload::Stream( - Box::pin(futures::stream::once(async { Ok(bytes) })) - ), - meta: meta.clone(), - range: 0..meta.size, + payload: GetResultPayload::Stream(Box::pin(futures::stream::once(async { Ok(bytes) }))), + meta: meta.clone(), + range: 0..meta.size, attributes: Default::default(), }) } else { @@ -485,10 +448,10 @@ impl DeltaCachedObjectStore { impl Clone for DeltaCachedObjectStore { fn clone(&self) -> Self { Self { - inner: self.inner.clone(), - cache: self.cache.clone(), - config: self.config.clone(), - metrics: self.metrics.clone(), + inner: self.inner.clone(), + cache: self.cache.clone(), + config: self.config.clone(), + metrics: self.metrics.clone(), access_patterns: self.access_patterns.clone(), } } @@ -498,47 +461,35 @@ impl Clone for DeltaCachedObjectStore { impl ObjectStore for DeltaCachedObjectStore { async fn put(&self, location: &Path, payload: PutPayload) -> ObjectStoreResult { let result = self.inner.put(location, payload).await?; - + // Invalidate cache for this path on writes if self.should_cache(location) { let cache_key = self.make_cache_key(location, None); let _ = self.cache.remove(&cache_key); debug!("Invalidated cache for: {}", location); } - + Ok(result) } - async fn put_opts( - &self, - location: &Path, - payload: PutPayload, - opts: PutOptions, - ) -> ObjectStoreResult { + async fn put_opts(&self, location: &Path, payload: PutPayload, opts: PutOptions) -> ObjectStoreResult { let result = self.inner.put_opts(location, payload, opts).await?; - + // Invalidate cache for this path on writes if self.should_cache(location) { let cache_key = self.make_cache_key(location, None); let _ = self.cache.remove(&cache_key); debug!("Invalidated cache for: {}", location); } - + Ok(result) } - async fn put_multipart( - &self, - location: &Path, - ) -> ObjectStoreResult> { + async fn put_multipart(&self, location: &Path) -> ObjectStoreResult> { self.inner.put_multipart(location).await } - async fn put_multipart_opts( - &self, - location: &Path, - opts: PutMultipartOpts, - ) -> ObjectStoreResult> { + async fn put_multipart_opts(&self, location: &Path, opts: PutMultipartOpts) -> ObjectStoreResult> { self.inner.put_multipart_opts(location, opts).await } @@ -566,14 +517,14 @@ impl ObjectStore for DeltaCachedObjectStore { async fn delete(&self, location: &Path) -> ObjectStoreResult<()> { let result = self.inner.delete(location).await; - + // Invalidate cache on delete if self.should_cache(location) { let cache_key = self.make_cache_key(location, None); let _ = self.cache.remove(&cache_key); debug!("Invalidated cache on delete: {}", location); } - + result } @@ -595,27 +546,27 @@ impl ObjectStore for DeltaCachedObjectStore { async fn copy(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { let result = self.inner.copy(from, to).await; - + // Invalidate cache for destination if self.should_cache(to) { let cache_key = self.make_cache_key(to, None); let _ = self.cache.remove(&cache_key); debug!("Invalidated cache on copy destination: {}", to); } - + result } async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { let result = self.inner.copy_if_not_exists(from, to).await; - + // Invalidate cache for destination if self.should_cache(to) { let cache_key = self.make_cache_key(to, None); let _ = self.cache.remove(&cache_key); debug!("Invalidated cache on conditional copy: {}", to); } - + result } } @@ -693,10 +644,7 @@ impl DeltaCacheBuilder { self } - pub async fn build( - self, - inner: Arc, - ) -> ObjectStoreResult> { + pub async fn build(self, inner: Arc) -> ObjectStoreResult> { let cached_store = DeltaCachedObjectStore::new(inner, self.config).await?; Ok(Arc::new(cached_store)) } @@ -706,4 +654,4 @@ impl Default for DeltaCacheBuilder { fn default() -> Self { Self::new() } -} \ No newline at end of file +} From 94c01e8c40ce3fdb8bc1e64daade2703cd2450ce Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 9 Jun 2025 15:50:13 +0100 Subject: [PATCH 25/33] formatting --- src/batch_queue.rs | 1 - src/database.rs | 2 -- src/delta.rs | 5 ++--- src/obj_store.rs | 6 +++--- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/batch_queue.rs b/src/batch_queue.rs index 6e7a53e1..f3afcf6e 100644 --- a/src/batch_queue.rs +++ b/src/batch_queue.rs @@ -109,7 +109,6 @@ mod tests { use std::sync::Arc; use chrono::Utc; - use serde_arrow::schema::SchemaLike; use tokio::time::sleep; use super::*; diff --git a/src/database.rs b/src/database.rs index 1d239911..169c0c6f 100644 --- a/src/database.rs +++ b/src/database.rs @@ -441,8 +441,6 @@ impl Database { // Records should be grouped by span, and separated into groups then inserted into the // correct table. - use serde_arrow::schema::SchemaLike; - // Convert OtelLogsAndSpans records to Arrow RecordBatch format let fields = OtelLogsAndSpans::fields()?; let batch = serde_arrow::to_record_batch(&fields, &records)?; diff --git a/src/delta.rs b/src/delta.rs index d2d78f2a..3d137d8b 100644 --- a/src/delta.rs +++ b/src/delta.rs @@ -3,15 +3,14 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use deltalake::{ DeltaTable, DeltaTableBuilder, DeltaTableError, arrow::record_batch::RecordBatch, - logstore::LogStore, operations::{create::CreateBuilder, write::WriteBuilder}, storage::ObjectStoreRef, }; -use object_store::{ObjectStore, aws::AmazonS3Builder, azure::MicrosoftAzureBuilder, gcp::GoogleCloudStorageBuilder, local::LocalFileSystem, memory::InMemory}; +use object_store::{aws::AmazonS3Builder, azure::MicrosoftAzureBuilder, gcp::GoogleCloudStorageBuilder, local::LocalFileSystem, memory::InMemory}; use tokio; use url::Url; -use crate::obj_store::{CacheMetrics, DeltaCacheBuilder, DeltaCacheConfig, DeltaCachedObjectStore}; +use crate::obj_store::{CacheMetrics, DeltaCacheBuilder, DeltaCacheConfig}; /// Helper struct for creating Delta tables with caching pub struct CachedDeltaTableBuilder { diff --git a/src/obj_store.rs b/src/obj_store.rs index ddf53c7c..fea557d6 100644 --- a/src/obj_store.rs +++ b/src/obj_store.rs @@ -1,17 +1,17 @@ use std::{ collections::HashMap, - fmt::{Debug, Display}, + fmt::Debug, ops::Range, sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; -use async_stream::{stream, try_stream}; +use async_stream::stream; use async_trait::async_trait; use bytes::Bytes; use chrono::DateTime; use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; -use futures::stream::{self, BoxStream, Once, StreamExt, TryStreamExt}; +use futures::stream::{BoxStream, StreamExt, TryStreamExt}; use object_store::{ GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult, path::Path, From 059f774f166a737700d91012e06f020b861f3202 Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 15 Jun 2025 21:08:03 +0100 Subject: [PATCH 26/33] tests passed --- Cargo.toml | 2 +- src/delta.rs | 159 ++++++++++++++++------------------------------- src/obj_store.rs | 10 +-- 3 files changed, 59 insertions(+), 112 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 14cbdf92..8fd11767 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ log = "0.4.25" color-eyre = "0.6.3" arrow-schema = "54.1.0" regex = "1.11.1" -deltalake = { version = "0.25.0", features = ["datafusion", "s3","azure", "gcs",] } +deltalake = { version = "0.26.2", features = ["datafusion", "s3","azure", "gcs",] } delta_kernel = { version = "0.8.0", features = [ "arrow-conversion", "default-engine", diff --git a/src/delta.rs b/src/delta.rs index 3d137d8b..9411507e 100644 --- a/src/delta.rs +++ b/src/delta.rs @@ -4,7 +4,7 @@ use deltalake::{ DeltaTable, DeltaTableBuilder, DeltaTableError, arrow::record_batch::RecordBatch, operations::{create::CreateBuilder, write::WriteBuilder}, - storage::ObjectStoreRef, + logstore::ObjectStoreRef, }; use object_store::{aws::AmazonS3Builder, azure::MicrosoftAzureBuilder, gcp::GoogleCloudStorageBuilder, local::LocalFileSystem, memory::InMemory}; use tokio; @@ -53,15 +53,17 @@ impl CachedDeltaTableBuilder { } /// Build the Delta table with caching - pub async fn build(self) -> Result { + pub async fn build(&self) -> Result { let base_store = self.create_base_object_store().await?; - let final_store: ObjectStoreRef = if let Some(cache_config) = self.cache_config { + let final_store: ObjectStoreRef = if let Some(cache_config) = &self.cache_config { // Wrap with cache - let cached_store = DeltaCacheBuilder::new() + + + (DeltaCacheBuilder::new() .with_memory_capacity(cache_config.memory_capacity) .with_disk_capacity(cache_config.disk_capacity) - .with_disk_path(cache_config.disk_cache_dir) + .with_disk_path(cache_config.disk_cache_dir.clone()) .with_ttl(Duration::from_secs(cache_config.ttl_seconds)) .with_compression(cache_config.compression_level) .enable_metrics(cache_config.enable_metrics) @@ -71,9 +73,7 @@ impl CachedDeltaTableBuilder { .cache_checkpoints(cache_config.cache_checkpoints) .build(base_store) .await - .map_err(|e| DeltaTableError::ObjectStore { source: e })?; - - cached_store + .map_err(|e| DeltaTableError::ObjectStore { source: e })?) as _ } else { base_store }; @@ -87,7 +87,7 @@ impl CachedDeltaTableBuilder { /// Create the base object store based on URI scheme async fn create_base_object_store(&self) -> Result { - let uri = Url::parse(&self.table_uri).map_err(|e| DeltaTableError::Generic(format!("Invalid URI: {}", e)))?; + let uri = Url::parse(&self.table_uri).map_err(|e| DeltaTableError::Generic(format!("Invalid URI: {e}")))?; match uri.scheme() { "s3" | "s3a" => { @@ -158,42 +158,12 @@ impl CachedDeltaTableBuilder { "memory" => Ok(Arc::new(InMemory::new())), - scheme => Err(DeltaTableError::Generic(format!("Unsupported scheme: {}", scheme))), + scheme => Err(DeltaTableError::Generic(format!("Unsupported scheme: {scheme}"))), } } } -/// Extension trait for DeltaTable to access cache metrics -pub trait DeltaTableCacheExt { - /// Get cache metrics if the table is using a cached store - async fn cache_metrics(&self) -> Option; - /// Get access patterns if the table is using a cached store - async fn access_patterns(&self) -> Option>; -} - -impl DeltaTableCacheExt for DeltaTable { - async fn cache_metrics(&self) -> Option { - // // Try to downcast the object store to our cached implementation - // let store = self.object_store(); - // if let Some(cached_store) = store{ - // Some(cached_store.metrics().await) - // } else { - // None - // } - todo!() - } - - async fn access_patterns(&self) -> Option> { - // let store = self.object_store(); - // if let Some(cached_store) = store { - // Some(cached_store.get_access_patterns().await) - // } else { - // None - // } - todo!() - } -} /// Convenience functions for common Delta operations with caching pub struct CachedDeltaOps; @@ -201,7 +171,7 @@ pub struct CachedDeltaOps; impl CachedDeltaOps { /// Create a new Delta table with caching enabled pub async fn create_table( - table_uri: &str, schema: arrow::datatypes::SchemaRef, cache_config: Option, + table_uri: &str, cache_config: Option, ) -> Result { let mut builder = CachedDeltaTableBuilder::new(table_uri); @@ -211,16 +181,13 @@ impl CachedDeltaOps { let table = builder.build().await?; - // Create the table if it doesn't exist - CreateBuilder::new() + CreateBuilder::new() .with_log_store(table.log_store()) .with_table_name(table_uri) - // .with_columns(schema.fields().iter().cloned()) - .await?; + + .await - // Reload to get the created table - // builder.build().await - todo!() + } /// Open an existing Delta table with caching @@ -251,6 +218,35 @@ impl CachedDeltaOps { } } +#[tokio::main] +async fn main() -> Result<(), Box> { + // Example 1: Simple cached Delta table + let cache_config = DeltaCacheConfig { + memory_capacity: 256 * 1024 * 1024, // 256MB + disk_capacity: 1024 * 1024 * 1024, // 1GB + disk_cache_dir: "/tmp/delta_cache".to_string(), + ttl_seconds: 3600, // 1 hour + enable_metrics: true, + enable_cache_warming: true, + ..Default::default() + }; + + let table = CachedDeltaTableBuilder::new("s3://my-bucket/my-table") + .with_cache_config(cache_config) + .with_storage_option("AWS_REGION", "us-west-2") + .with_storage_option("AWS_ACCESS_KEY_ID", "your-access-key") + .with_storage_option("AWS_SECRET_ACCESS_KEY", "your-secret-key") + .build() + .await?; + + println!("Table loaded with {} files", table.get_files_count()); + + + + + Ok(()) +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -285,15 +281,12 @@ mod tests { }; // Create table with caching - let table = CachedDeltaOps::create_table(&table_uri, schema.clone(), Some(cache_config)).await.unwrap(); + let table = CachedDeltaOps::create_table(&table_uri, Some(cache_config)).await.unwrap(); // Verify the table was created assert!(table.get_files_count() == 0); // New table, no data files yet - // Check if cache metrics are available - if let Some(metrics) = table.cache_metrics().await { - println!("Cache metrics: {:?}", metrics); - } + } #[tokio::test] @@ -308,7 +301,7 @@ mod tests { ])); // Create table with cache - let mut table = CachedDeltaOps::create_table(&table_uri, schema.clone(), Some(DeltaCacheConfig::default())).await.unwrap(); + let mut table = CachedDeltaOps::create_table(&table_uri, Some(DeltaCacheConfig::default())).await.unwrap(); // Create some test data let batch = RecordBatch::try_new( @@ -322,64 +315,18 @@ mod tests { // Read data back (should hit cache on subsequent reads) let files = table.get_file_uris(); - assert!(!files.is_err()); + assert!(files.is_ok()); - // Check cache metrics - if let Some(metrics) = table.cache_metrics().await { - println!("After write - Cache metrics: {:?}", metrics); - } + // Read again to test cache hit let _files_again = table.get_file_uris(); - if let Some(metrics) = table.cache_metrics().await { - println!("After second read - Cache metrics: {:?}", metrics); - assert!(metrics.total_requests > 0); - } - } -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - // Example 1: Simple cached Delta table - let cache_config = DeltaCacheConfig { - memory_capacity: 256 * 1024 * 1024, // 256MB - disk_capacity: 1024 * 1024 * 1024, // 1GB - disk_cache_dir: "/tmp/delta_cache".to_string(), - ttl_seconds: 3600, // 1 hour - enable_metrics: true, - enable_cache_warming: true, - ..Default::default() - }; - - let table = CachedDeltaTableBuilder::new("s3://my-bucket/my-table") - .with_cache_config(cache_config) - .with_storage_option("AWS_REGION", "us-west-2") - .with_storage_option("AWS_ACCESS_KEY_ID", "your-access-key") - .with_storage_option("AWS_SECRET_ACCESS_KEY", "your-secret-key") - .build() - .await?; - - println!("Table loaded with {} files", table.get_files_count()); - - // Check cache performance - if let Some(metrics) = table.cache_metrics().await { - println!("Cache hit rate: {:.2}%", metrics.hit_rate() * 100.0); - println!("Total requests: {}", metrics.total_requests); - println!("Cache hits: {}", metrics.hits); - println!("Cache misses: {}", metrics.misses); + } - // Example 2: Monitor access patterns - if let Some(patterns) = table.access_patterns().await { - println!("Most accessed files:"); - let mut sorted_patterns: Vec<_> = patterns.iter().collect(); - sorted_patterns.sort_by(|a, b| b.1.cmp(a.1)); - - for (path, count) in sorted_patterns.iter().take(10) { - println!(" {}: {} accesses", path, count); - } + #[tokio::test] + async fn test_write_and_read_with_caches3() { + let _s= main(); } - - Ok(()) } diff --git a/src/obj_store.rs b/src/obj_store.rs index fea557d6..d12050af 100644 --- a/src/obj_store.rs +++ b/src/obj_store.rs @@ -364,7 +364,7 @@ impl DeltaCachedObjectStore { } else { debug!("Cache invalidated due to metadata mismatch: {}", location); // Remove stale entry - let _ = self.cache.remove(&cache_key); + self.cache.remove(&cache_key); } } Err(e) => { @@ -378,11 +378,11 @@ impl DeltaCachedObjectStore { meta: ObjectMeta { location: location.clone(), last_modified: DateTime::::MIN_UTC, - size: cached_obj.original_size, + size: cached_obj.original_size as u64, e_tag: cached_obj.etag.clone(), version: None, }, - range: 0..cached_obj.original_size, + range: 0..cached_obj.original_size as u64, attributes: Default::default(), }); } @@ -402,7 +402,7 @@ impl DeltaCachedObjectStore { let meta = result.meta.clone(); // Only cache if object size is within limits - if meta.size <= self.config.max_object_size { + if meta.size <= self.config.max_object_size as u64 { // Read the entire payload for caching let bytes = result.bytes().await?; @@ -501,7 +501,7 @@ impl ObjectStore for DeltaCachedObjectStore { self.get_with_cache(location, options).await } - async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { + async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { let options = GetOptions { range: Some(GetRange::Bounded(range)), ..Default::default() From 6edcef061df30907a58b7be35c1fec745aad3711 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 16 Jun 2025 11:31:42 +0100 Subject: [PATCH 27/33] fix : versioning errors --- Cargo.lock | 1707 ++++++++++++++++++++++++++++++++++++++++------------ Cargo.toml | 4 +- 2 files changed, 1313 insertions(+), 398 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1414e9f7..6dc70a2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,45 +409,97 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc208515aa0151028e464cc94a692156e945ce5126abd3537bb7fd6ba2143ed1" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 54.3.1", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.3.1", + "arrow-csv 54.3.1", + "arrow-data 54.3.1", + "arrow-ipc 54.3.1", + "arrow-json 54.3.1", + "arrow-ord 54.2.1", + "arrow-row 54.2.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "arrow-string 54.2.1", +] + +[[package]] +name = "arrow" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3095aaf545942ff5abd46654534f15b03a90fba78299d661e045e5d587222f0d" +dependencies = [ + "arrow-arith 55.0.0", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", + "arrow-csv 55.0.0", + "arrow-data 55.0.0", + "arrow-ipc 55.0.0", + "arrow-json 55.0.0", + "arrow-ord 55.0.0", + "arrow-row 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", + "arrow-string 55.0.0", ] [[package]] name = "arrow-arith" -version = "54.2.1" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e07e726e2b3f7816a85c6a45b6ec118eeeabf0b2a8c208122ad949437181f49a" +checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "num", +] + +[[package]] +name = "arrow-arith" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00752064ff47cee746e816ddb8450520c3a52cbad1e256f6fa861a35f86c45e7" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "chrono", "num", ] [[package]] name = "arrow-array" -version = "54.2.1" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2262eba4f16c78496adfd559a29fe4b24df6088efc9985a873d58e92be022d5" +checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" dependencies = [ "ahash 0.8.11", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.15.2", + "num", +] + +[[package]] +name = "arrow-array" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cebfe926794fbc1f49ddd0cdaf898956ca9f6e79541efce62dabccfd81380472" +dependencies = [ + "ahash 0.8.11", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "chrono", "chrono-tz", "half", @@ -466,17 +518,49 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0303c7ec4cf1a2c60310fc4d6bbc3350cd051a17bf9e9c0a8e47b4db79277824" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-cast" -version = "54.2.1" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-cast" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4103d88c5b441525ed4ac23153be7458494c2b0c9a11115848fdb9b81f6f886a" +checksum = "335f769c5a218ea823d3760a743feba1ef7857cba114c01399a891c2fff34285" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", "atoi", "base64 0.22.1", "chrono", @@ -489,13 +573,29 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "54.2.1" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d3cb0914486a3cae19a5cad2598e44e225d53157926d0ada03c20521191a65" +checksum = "1644877d8bc9a0ef022d9153dc29375c2bda244c39aec05a91d0e87ccf77995f" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 54.3.1", + "arrow-cast 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "csv", + "csv-core", + "lazy_static", + "regex 1.11.1", +] + +[[package]] +name = "arrow-csv" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "510db7dfbb4d5761826516cc611d97b3a68835d0ece95b034a052601109c0b1b" +dependencies = [ + "arrow-array 55.0.0", + "arrow-cast 55.0.0", + "arrow-schema 55.0.0", "chrono", "csv", "csv-core", @@ -509,44 +609,94 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 54.3.1", + "arrow-schema 54.3.1", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8affacf3351a24039ea24adab06f316ded523b6f8c3dbe28fbac5f18743451b" +dependencies = [ + "arrow-buffer 55.0.0", + "arrow-schema 55.0.0", "half", "num", ] [[package]] name = "arrow-ipc" -version = "54.2.1" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddecdeab02491b1ce88885986e25002a3da34dd349f682c7cfe67bab7cc17b86" +checksum = "62ff528658b521e33905334723b795ee56b393dbe9cf76c8b1f64b648c65a60c" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "flatbuffers", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "flatbuffers 24.12.23", + "lz4_flex", +] + +[[package]] +name = "arrow-ipc" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69880a9e6934d9cba2b8630dd08a3463a91db8693b16b499d54026b6137af284" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "flatbuffers 25.2.10", "lz4_flex", ] [[package]] name = "arrow-json" -version = "54.2.1" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee5b4ca98a7fb2efb9ab3309a5d1c88b5116997ff93f3147efdc1062a6158e9" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "half", + "indexmap 2.9.0", + "lexical-core", + "memchr", + "num", + "serde", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-json" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b9340013413eb84868682ace00a1098c81a5ebc96d279f7ebf9a4cac3c0fd" +checksum = "d8dafd17a05449e31e0114d740530e0ada7379d7cb9c338fd65b09a8130960b0" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "chrono", "half", "indexmap 2.9.0", "lexical-core", + "memchr", "num", "serde", "serde_json", + "simdutf8", ] [[package]] @@ -555,11 +705,24 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f841bfcc1997ef6ac48ee0305c4dfceb1f7c786fe31e67c1186edf775e1f1160" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", +] + +[[package]] +name = "arrow-ord" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "895644523af4e17502d42c3cb6b27cb820f0cb77954c22d75c23a85247c849e1" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", ] [[package]] @@ -568,10 +731,23 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eeb55b0a0a83851aa01f2ca5ee5648f607e8506ba6802577afdda9d75cdedcd" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "half", +] + +[[package]] +name = "arrow-row" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be8a2a4e5e7d9c822b2b8095ecd77010576d824f654d347817640acfc97d229" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "half", ] @@ -580,6 +756,12 @@ name = "arrow-schema" version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" + +[[package]] +name = "arrow-schema" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49" dependencies = [ "bitflags 2.9.0", "serde", @@ -587,15 +769,29 @@ dependencies = [ [[package]] name = "arrow-select" -version = "54.2.1" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e2932aece2d0c869dd2125feb9bd1709ef5c445daa3838ac4112dcfa0fda52c" +checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "num", +] + +[[package]] +name = "arrow-select" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa5f5a93c75f46ef48e4001535e7b6c922eeb0aa20b73cf58d09e13d057490d8" +dependencies = [ + "ahash 0.8.11", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", "num", ] @@ -605,11 +801,28 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "912e38bd6a7a7714c1d9b61df80315685553b7455e8a6045c27531d8ecd5b458" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "memchr", + "num", + "regex 1.11.1", + "regex-syntax 0.8.5", +] + +[[package]] +name = "arrow-string" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7005d858d84b56428ba2a98a107fe88c0132c61793cf6b8232a1f9bfc0452b" +dependencies = [ + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-data 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", "memchr", "num", "regex 1.11.1", @@ -1553,9 +1766,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1563,7 +1776,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -1783,6 +1996,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -2134,40 +2356,40 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "914e6f9525599579abbd90b0f7a55afcaaaa40350b9e9ed52563f126dfe45fd3" dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", + "arrow 54.2.1", + "arrow-ipc 54.3.1", + "arrow-schema 54.3.1", "async-trait", "bytes", "bzip2", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-macros", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-sql", + "datafusion-catalog 46.0.1", + "datafusion-catalog-listing 46.0.1", + "datafusion-common 46.0.1", + "datafusion-common-runtime 46.0.1", + "datafusion-datasource 46.0.1", + "datafusion-execution 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-expr-common 46.0.1", + "datafusion-functions 46.0.1", + "datafusion-functions-aggregate 46.0.1", + "datafusion-functions-nested 46.0.1", + "datafusion-functions-table 46.0.1", + "datafusion-functions-window 46.0.1", + "datafusion-macros 46.0.1", + "datafusion-optimizer 46.0.1", + "datafusion-physical-expr 46.0.1", + "datafusion-physical-expr-common 46.0.1", + "datafusion-physical-optimizer 46.0.1", + "datafusion-physical-plan 46.0.1", + "datafusion-sql 46.0.1", "flate2", "futures", "itertools 0.14.0", "log 0.4.27", - "object_store", + "object_store 0.11.2", "parking_lot 0.12.3", - "parquet", + "parquet 54.3.1", "rand 0.8.5", "regex 1.11.1", "sqlparser 0.54.0", @@ -2179,45 +2401,149 @@ dependencies = [ "zstd", ] +[[package]] +name = "datafusion" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" +dependencies = [ + "arrow 55.0.0", + "arrow-ipc 55.0.0", + "arrow-schema 55.0.0", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-catalog 47.0.0", + "datafusion-catalog-listing 47.0.0", + "datafusion-common 47.0.0", + "datafusion-common-runtime 47.0.0", + "datafusion-datasource 47.0.0", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-expr-common 47.0.0", + "datafusion-functions 47.0.0", + "datafusion-functions-aggregate 47.0.0", + "datafusion-functions-nested 47.0.0", + "datafusion-functions-table 47.0.0", + "datafusion-functions-window 47.0.0", + "datafusion-macros 47.0.0", + "datafusion-optimizer 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", + "datafusion-physical-optimizer 47.0.0", + "datafusion-physical-plan 47.0.0", + "datafusion-session", + "datafusion-sql 47.0.0", + "flate2", + "futures", + "itertools 0.14.0", + "log 0.4.27", + "object_store 0.12.2", + "parking_lot 0.12.3", + "parquet 55.0.0", + "rand 0.8.5", + "regex 1.11.1", + "sqlparser 0.55.0", + "tempfile", + "tokio", + "url", + "uuid", + "xz2", + "zstd", +] + [[package]] name = "datafusion-catalog" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "998a6549e6ee4ee3980e05590b2960446a56b343ea30199ef38acd0e0b9036e2" dependencies = [ - "arrow", + "arrow 54.2.1", "async-trait", "dashmap", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", - "datafusion-sql", + "datafusion-common 46.0.1", + "datafusion-execution 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-physical-plan 46.0.1", + "datafusion-sql 46.0.1", "futures", "itertools 0.14.0", "log 0.4.27", "parking_lot 0.12.3", ] +[[package]] +name = "datafusion-catalog" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "dashmap", + "datafusion-common 47.0.0", + "datafusion-common-runtime 47.0.0", + "datafusion-datasource 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-plan 47.0.0", + "datafusion-session", + "datafusion-sql 47.0.0", + "futures", + "itertools 0.14.0", + "log 0.4.27", + "object_store 0.12.2", + "parking_lot 0.12.3", + "tokio", +] + [[package]] name = "datafusion-catalog-listing" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5ac10096a5b3c0d8a227176c0e543606860842e943594ccddb45cf42a526e43" dependencies = [ - "arrow", + "arrow 54.2.1", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-catalog 46.0.1", + "datafusion-common 46.0.1", + "datafusion-datasource 46.0.1", + "datafusion-execution 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-physical-expr 46.0.1", + "datafusion-physical-expr-common 46.0.1", + "datafusion-physical-plan 46.0.1", "futures", "log 0.4.27", - "object_store", + "object_store 0.11.2", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "datafusion-catalog 47.0.0", + "datafusion-common 47.0.0", + "datafusion-datasource 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", + "datafusion-physical-plan 47.0.0", + "datafusion-session", + "futures", + "log 0.4.27", + "object_store 0.12.2", "tokio", ] @@ -2228,16 +2554,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f53d7ec508e1b3f68bd301cee3f649834fad51eff9240d898a4b2614cfd0a7a" dependencies = [ "ahash 0.8.11", - "arrow", - "arrow-ipc", + "arrow 54.2.1", + "arrow-ipc 54.3.1", "base64 0.22.1", "half", "hashbrown 0.14.5", "indexmap 2.9.0", "libc", "log 0.4.27", - "object_store", - "parquet", + "object_store 0.11.2", + "parquet 54.3.1", "paste", "recursive", "sqlparser 0.54.0", @@ -2245,6 +2571,30 @@ dependencies = [ "web-time", ] +[[package]] +name = "datafusion-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" +dependencies = [ + "ahash 0.8.11", + "arrow 55.0.0", + "arrow-ipc 55.0.0", + "base64 0.22.1", + "half", + "hashbrown 0.14.5", + "indexmap 2.9.0", + "libc", + "log 0.4.27", + "object_store 0.12.2", + "parquet 55.0.0", + "paste", + "recursive", + "sqlparser 0.55.0", + "tokio", + "web-time", +] + [[package]] name = "datafusion-common-runtime" version = "46.0.1" @@ -2255,32 +2605,43 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-common-runtime" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5167bb1d2ccbb87c6bc36c295274d7a0519b14afcfdaf401d53cbcaa4ef4968b" +dependencies = [ + "futures", + "log 0.4.27", + "tokio", +] + [[package]] name = "datafusion-datasource" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf7f37ad8b6e88b46c7eeab3236147d32ea64b823544f498455a8d9042839c92" dependencies = [ - "arrow", + "arrow 54.2.1", "async-compression", "async-trait", "bytes", "bzip2", "chrono", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-catalog 46.0.1", + "datafusion-common 46.0.1", + "datafusion-common-runtime 46.0.1", + "datafusion-execution 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-physical-expr 46.0.1", + "datafusion-physical-expr-common 46.0.1", + "datafusion-physical-plan 46.0.1", "flate2", "futures", "glob", "itertools 0.14.0", "log 0.4.27", - "object_store", + "object_store 0.11.2", "rand 0.8.5", "tokio", "tokio-util", @@ -2289,25 +2650,167 @@ dependencies = [ "zstd", ] +[[package]] +name = "datafusion-datasource" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" +dependencies = [ + "arrow 55.0.0", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common 47.0.0", + "datafusion-common-runtime 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", + "datafusion-physical-plan 47.0.0", + "datafusion-session", + "flate2", + "futures", + "glob", + "itertools 0.14.0", + "log 0.4.27", + "object_store 0.12.2", + "parquet 55.0.0", + "rand 0.8.5", + "tempfile", + "tokio", + "tokio-util", + "url", + "xz2", + "zstd", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "bytes", + "datafusion-catalog 47.0.0", + "datafusion-common 47.0.0", + "datafusion-common-runtime 47.0.0", + "datafusion-datasource 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", + "datafusion-physical-plan 47.0.0", + "datafusion-session", + "futures", + "object_store 0.12.2", + "regex 1.11.1", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "bytes", + "datafusion-catalog 47.0.0", + "datafusion-common 47.0.0", + "datafusion-common-runtime 47.0.0", + "datafusion-datasource 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", + "datafusion-physical-plan 47.0.0", + "datafusion-session", + "futures", + "object_store 0.12.2", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "bytes", + "datafusion-catalog 47.0.0", + "datafusion-common 47.0.0", + "datafusion-common-runtime 47.0.0", + "datafusion-datasource 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-functions-aggregate 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", + "datafusion-physical-optimizer 47.0.0", + "datafusion-physical-plan 47.0.0", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log 0.4.27", + "object_store 0.12.2", + "parking_lot 0.12.3", + "parquet 55.0.0", + "rand 0.8.5", + "tokio", +] + [[package]] name = "datafusion-doc" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7db7a0239fd060f359dc56c6e7db726abaa92babaed2fb2e91c3a8b2fff8b256" +[[package]] +name = "datafusion-doc" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91f8c2c5788ef32f48ff56c68e5b545527b744822a284373ac79bba1ba47292" + [[package]] name = "datafusion-execution" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0938f9e5b6bc5782be4111cdfb70c02b7b5451bf34fd57e4de062a7f7c4e31f1" dependencies = [ - "arrow", + "arrow 54.2.1", "dashmap", - "datafusion-common", - "datafusion-expr", + "datafusion-common 46.0.1", + "datafusion-expr 46.0.1", "futures", "log 0.4.27", - "object_store", + "object_store 0.11.2", + "parking_lot 0.12.3", + "rand 0.8.5", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-execution" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" +dependencies = [ + "arrow 55.0.0", + "dashmap", + "datafusion-common 47.0.0", + "datafusion-expr 47.0.0", + "futures", + "log 0.4.27", + "object_store 0.12.2", "parking_lot 0.12.3", "rand 0.8.5", "tempfile", @@ -2320,14 +2823,14 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b36c28b00b00019a8695ad7f1a53ee1673487b90322ecbd604e2cf32894eb14f" dependencies = [ - "arrow", + "arrow 54.2.1", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 46.0.1", + "datafusion-doc 46.0.1", + "datafusion-expr-common 46.0.1", + "datafusion-functions-aggregate-common 46.0.1", + "datafusion-functions-window-common 46.0.1", + "datafusion-physical-expr-common 46.0.1", "indexmap 2.9.0", "paste", "recursive", @@ -2335,14 +2838,48 @@ dependencies = [ "sqlparser 0.54.0", ] +[[package]] +name = "datafusion-expr" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" +dependencies = [ + "arrow 55.0.0", + "chrono", + "datafusion-common 47.0.0", + "datafusion-doc 47.0.0", + "datafusion-expr-common 47.0.0", + "datafusion-functions-aggregate-common 47.0.0", + "datafusion-functions-window-common 47.0.0", + "datafusion-physical-expr-common 47.0.0", + "indexmap 2.9.0", + "paste", + "recursive", + "serde_json", + "sqlparser 0.55.0", +] + [[package]] name = "datafusion-expr-common" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e" dependencies = [ - "arrow", - "datafusion-common", + "arrow 54.2.1", + "datafusion-common 46.0.1", + "indexmap 2.9.0", + "itertools 0.14.0", + "paste", +] + +[[package]] +name = "datafusion-expr-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" +dependencies = [ + "arrow 55.0.0", + "datafusion-common 47.0.0", "indexmap 2.9.0", "itertools 0.14.0", "paste", @@ -2354,18 +2891,47 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3196e37d7b65469fb79fee4f05e5bb58a456831035f9a38aa5919aeb3298d40" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 54.2.1", + "arrow-buffer 54.3.1", "base64 0.22.1", "blake2", "blake3", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 46.0.1", + "datafusion-doc 46.0.1", + "datafusion-execution 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-expr-common 46.0.1", + "datafusion-macros 46.0.1", + "hex", + "itertools 0.14.0", + "log 0.4.27", + "md-5", + "rand 0.8.5", + "regex 1.11.1", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" +dependencies = [ + "arrow 55.0.0", + "arrow-buffer 55.0.0", + "base64 0.22.1", + "blake2", + "blake3", + "chrono", + "datafusion-common 47.0.0", + "datafusion-doc 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-expr-common 47.0.0", + "datafusion-macros 47.0.0", "hex", "itertools 0.14.0", "log 0.4.27", @@ -2384,15 +2950,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adfc2d074d5ee4d9354fdcc9283d5b2b9037849237ddecb8942a29144b77ca05" dependencies = [ "ahash 0.8.11", - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "arrow 54.2.1", + "datafusion-common 46.0.1", + "datafusion-doc 46.0.1", + "datafusion-execution 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-functions-aggregate-common 46.0.1", + "datafusion-macros 46.0.1", + "datafusion-physical-expr 46.0.1", + "datafusion-physical-expr-common 46.0.1", + "half", + "log 0.4.27", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" +dependencies = [ + "ahash 0.8.11", + "arrow 55.0.0", + "datafusion-common 47.0.0", + "datafusion-doc 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-functions-aggregate-common 47.0.0", + "datafusion-macros 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", "half", "log 0.4.27", "paste", @@ -2405,10 +2992,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cbceba0f98d921309a9121b702bcd49289d383684cccabf9a92cda1602f3bbb" dependencies = [ "ahash 0.8.11", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "arrow 54.2.1", + "datafusion-common 46.0.1", + "datafusion-expr-common 46.0.1", + "datafusion-physical-expr-common 46.0.1", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" +dependencies = [ + "ahash 0.8.11", + "arrow 55.0.0", + "datafusion-common 47.0.0", + "datafusion-expr-common 47.0.0", + "datafusion-physical-expr-common 47.0.0", ] [[package]] @@ -2417,7 +3017,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9658d1ad5c3ac21667d04d01222202cb644fd85b2c5ea9d82c4efa33153d90" dependencies = [ - "datafusion", + "datafusion 46.0.1", "jiter", "log 0.4.27", "paste", @@ -2429,16 +3029,37 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "170e27ce4baa27113ddf5f77f1a7ec484b0dbeda0c7abbd4bad3fc609c8ab71a" dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-macros", - "datafusion-physical-expr-common", + "arrow 54.2.1", + "arrow-ord 54.2.1", + "datafusion-common 46.0.1", + "datafusion-doc 46.0.1", + "datafusion-execution 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-functions 46.0.1", + "datafusion-functions-aggregate 46.0.1", + "datafusion-macros 46.0.1", + "datafusion-physical-expr-common 46.0.1", + "itertools 0.14.0", + "log 0.4.27", + "paste", +] + +[[package]] +name = "datafusion-functions-nested" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" +dependencies = [ + "arrow 55.0.0", + "arrow-ord 55.0.0", + "datafusion-common 47.0.0", + "datafusion-doc 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-functions 47.0.0", + "datafusion-functions-aggregate 47.0.0", + "datafusion-macros 47.0.0", + "datafusion-physical-expr-common 47.0.0", "itertools 0.14.0", "log 0.4.27", "paste", @@ -2450,12 +3071,28 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d3a06a7f0817ded87b026a437e7e51de7f59d48173b0a4e803aa896a7bd6bb5" dependencies = [ - "arrow", + "arrow 54.2.1", + "async-trait", + "datafusion-catalog 46.0.1", + "datafusion-common 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-physical-plan 46.0.1", + "parking_lot 0.12.3", + "paste", +] + +[[package]] +name = "datafusion-functions-table" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" +dependencies = [ + "arrow 55.0.0", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-catalog 47.0.0", + "datafusion-common 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-physical-plan 47.0.0", "parking_lot 0.12.3", "paste", ] @@ -2466,13 +3103,30 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6c608b66496a1e05e3d196131eb9bebea579eed1f59e88d962baf3dda853bc6" dependencies = [ - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 46.0.1", + "datafusion-doc 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-functions-window-common 46.0.1", + "datafusion-macros 46.0.1", + "datafusion-physical-expr 46.0.1", + "datafusion-physical-expr-common 46.0.1", + "log 0.4.27", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefc2d77646e1aadd1d6a9c40088937aedec04e68c5f0465939912e1291f8193" +dependencies = [ + "datafusion-common 47.0.0", + "datafusion-doc 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-functions-window-common 47.0.0", + "datafusion-macros 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", "log 0.4.27", "paste", ] @@ -2483,8 +3137,18 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da2f9d83348957b4ad0cd87b5cb9445f2651863a36592fe5484d43b49a5f8d82" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 46.0.1", + "datafusion-physical-expr-common 46.0.1", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4aff082c42fa6da99ce0698c85addd5252928c908eb087ca3cfa64ff16b313" +dependencies = [ + "datafusion-common 47.0.0", + "datafusion-physical-expr-common 47.0.0", ] [[package]] @@ -2493,7 +3157,18 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4800e1ff7ecf8f310887e9b54c9c444b8e215ccbc7b21c2f244cfae373b1ece7" dependencies = [ - "datafusion-expr", + "datafusion-expr 46.0.1", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "datafusion-macros" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" +dependencies = [ + "datafusion-expr 47.0.0", "quote", "syn 2.0.100", ] @@ -2504,32 +3179,73 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "971c51c54cd309001376fae752fb15a6b41750b6d1552345c46afbfb6458801b" dependencies = [ - "arrow", + "arrow 54.2.1", + "chrono", + "datafusion-common 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-physical-expr 46.0.1", + "indexmap 2.9.0", + "itertools 0.14.0", + "log 0.4.27", + "recursive", + "regex 1.11.1", + "regex-syntax 0.8.5", +] + +[[package]] +name = "datafusion-optimizer" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" +dependencies = [ + "arrow 55.0.0", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-common 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-physical-expr 47.0.0", + "indexmap 2.9.0", + "itertools 0.14.0", + "log 0.4.27", + "recursive", + "regex 1.11.1", + "regex-syntax 0.8.5", +] + +[[package]] +name = "datafusion-physical-expr" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85" +dependencies = [ + "ahash 0.8.11", + "arrow 54.2.1", + "datafusion-common 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-expr-common 46.0.1", + "datafusion-functions-aggregate-common 46.0.1", + "datafusion-physical-expr-common 46.0.1", + "half", + "hashbrown 0.14.5", "indexmap 2.9.0", "itertools 0.14.0", "log 0.4.27", - "recursive", - "regex 1.11.1", - "regex-syntax 0.8.5", + "paste", + "petgraph", ] [[package]] name = "datafusion-physical-expr" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85" +checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" dependencies = [ "ahash 0.8.11", - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "arrow 55.0.0", + "datafusion-common 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-expr-common 47.0.0", + "datafusion-functions-aggregate-common 47.0.0", + "datafusion-physical-expr-common 47.0.0", "half", "hashbrown 0.14.5", "indexmap 2.9.0", @@ -2546,9 +3262,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f8c25dcd069073a75b3d2840a79d0f81e64bdd2c05f2d3d18939afb36a7dcb" dependencies = [ "ahash 0.8.11", - "arrow", - "datafusion-common", - "datafusion-expr-common", + "arrow 54.2.1", + "datafusion-common 46.0.1", + "datafusion-expr-common 46.0.1", + "hashbrown 0.14.5", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" +dependencies = [ + "ahash 0.8.11", + "arrow 55.0.0", + "datafusion-common 47.0.0", + "datafusion-expr-common 47.0.0", "hashbrown 0.14.5", "itertools 0.14.0", ] @@ -2559,14 +3289,33 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68da5266b5b9847c11d1b3404ee96b1d423814e1973e1ad3789131e5ec912763" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "arrow 54.2.1", + "datafusion-common 46.0.1", + "datafusion-execution 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-expr-common 46.0.1", + "datafusion-physical-expr 46.0.1", + "datafusion-physical-expr-common 46.0.1", + "datafusion-physical-plan 46.0.1", + "itertools 0.14.0", + "log 0.4.27", + "recursive", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" +dependencies = [ + "arrow 55.0.0", + "datafusion-common 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-expr-common 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", + "datafusion-physical-plan 47.0.0", "itertools 0.14.0", "log 0.4.27", "recursive", @@ -2579,18 +3328,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88cc160df00e413e370b3b259c8ea7bfbebc134d32de16325950e9e923846b7f" dependencies = [ "ahash 0.8.11", - "arrow", - "arrow-ord", - "arrow-schema", + "arrow 54.2.1", + "arrow-ord 54.2.1", + "arrow-schema 54.3.1", + "async-trait", + "chrono", + "datafusion-common 46.0.1", + "datafusion-common-runtime 46.0.1", + "datafusion-execution 46.0.1", + "datafusion-expr 46.0.1", + "datafusion-functions-window-common 46.0.1", + "datafusion-physical-expr 46.0.1", + "datafusion-physical-expr-common 46.0.1", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap 2.9.0", + "itertools 0.14.0", + "log 0.4.27", + "parking_lot 0.12.3", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-physical-plan" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" +dependencies = [ + "ahash 0.8.11", + "arrow 55.0.0", + "arrow-ord 55.0.0", + "arrow-schema 55.0.0", "async-trait", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 47.0.0", + "datafusion-common-runtime 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-functions-window-common 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-expr-common 47.0.0", "futures", "half", "hashbrown 0.14.5", @@ -2609,48 +3388,72 @@ source = "git+https://github.com/sunng87/datafusion-postgres.git?rev=2cf58787a8b dependencies = [ "async-trait", "chrono", - "datafusion", + "datafusion 46.0.1", "futures", "pgwire", ] [[package]] name = "datafusion-proto" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f6ef4c6eb52370cb48639e25e2331a415aac0b2b0a0a472b36e26603bdf184f" +checksum = "a4a1afb2bdb05de7ff65be6883ebfd4ec027bd9f1f21c46aa3afd01927160a83" dependencies = [ - "arrow", + "arrow 55.0.0", "chrono", - "datafusion", - "datafusion-common", - "datafusion-expr", + "datafusion 47.0.0", + "datafusion-common 47.0.0", + "datafusion-expr 47.0.0", "datafusion-proto-common", - "object_store", + "object_store 0.12.2", "prost", ] [[package]] name = "datafusion-proto-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5faf4a9bbb0d0a305fea8a6db21ba863286b53e53a212e687d2774028dd6f03f" +checksum = "35b7a5876ebd6b564fb9a1fd2c3a2a9686b787071a256b47e4708f0916f9e46f" dependencies = [ - "arrow", - "datafusion-common", + "arrow 55.0.0", + "datafusion-common 47.0.0", "prost", ] +[[package]] +name = "datafusion-session" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" +dependencies = [ + "arrow 55.0.0", + "async-trait", + "dashmap", + "datafusion-common 47.0.0", + "datafusion-common-runtime 47.0.0", + "datafusion-execution 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-plan 47.0.0", + "datafusion-sql 47.0.0", + "futures", + "itertools 0.14.0", + "log 0.4.27", + "object_store 0.12.2", + "parking_lot 0.12.3", + "tokio", +] + [[package]] name = "datafusion-sql" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325a212b67b677c0eb91447bf9a11b630f9fc4f62d8e5d145bf859f5a6b29e64" dependencies = [ - "arrow", + "arrow 54.2.1", "bigdecimal", - "datafusion-common", - "datafusion-expr", + "datafusion-common 46.0.1", + "datafusion-expr 46.0.1", "indexmap 2.9.0", "log 0.4.27", "recursive", @@ -2658,6 +3461,23 @@ dependencies = [ "sqlparser 0.54.0", ] +[[package]] +name = "datafusion-sql" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" +dependencies = [ + "arrow 55.0.0", + "bigdecimal", + "datafusion-common 47.0.0", + "datafusion-expr 47.0.0", + "indexmap 2.9.0", + "log 0.4.27", + "recursive", + "regex 1.11.1", + "sqlparser 0.55.0", +] + [[package]] name = "datafusion-uwheel" version = "46.0.0" @@ -2665,47 +3485,81 @@ source = "git+https://github.com/apitoolkit/datafusion-uwheel.git?branch=datafus dependencies = [ "bitpacking", "chrono", - "datafusion", + "datafusion 46.0.1", "uwheel", ] [[package]] name = "delta_kernel" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae7dc3012ad01882cd7669fd9524d7069cd5a6f12d69932a6f125d3bf503019" +checksum = "c96f51383ba327a1403e6e3458f8fc979d09d7200af56fa32681619f6c760dee" dependencies = [ - "arrow", + "arrow 55.0.0", "bytes", "chrono", - "delta_kernel_derive", - "fix-hidden-lifetime-bug", + "delta_kernel_derive 0.10.0", "futures", - "home", "indexmap 2.9.0", - "itertools 0.13.0", - "object_store", - "parquet", + "itertools 0.14.0", + "object_store 0.12.2", + "parquet 55.0.0", + "reqwest", + "roaring", + "rustc_version", + "serde", + "serde_json", + "strum", + "thiserror", + "tokio", + "tracing", + "url", + "uuid", + "z85", +] + +[[package]] +name = "delta_kernel" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0b553d03fce69da6bedd91ec7e2348d52af2783a95d2dc91970df0cb614783" +dependencies = [ + "bytes", + "chrono", + "delta_kernel_derive 0.12.1", + "futures", + "indexmap 2.9.0", + "itertools 0.14.0", "reqwest", "roaring", "rustc_version", "serde", "serde_json", "strum", - "thiserror 1.0.69", + "thiserror", "tokio", "tracing", "url", "uuid", - "visibility", "z85", ] [[package]] name = "delta_kernel_derive" -version = "0.8.0" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b49a2e67ebafbe644e36f251ee985f237bfb39e4ef1e312eb5876535bc449e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "delta_kernel_derive" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8e41236d5a9f04da3072d7186a76aba734e7bfd2cd05f7877fde172b65fb11" +checksum = "deacb882456b0a3e7a5bf22a708190758cc8f572e02cd34954931e24286a4509" dependencies = [ "proc-macro2", "quote", @@ -2714,9 +3568,9 @@ dependencies = [ [[package]] name = "deltalake" -version = "0.25.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78889f4005974b848f130fa5dedae81987f1bc93b107291ea87d900c93b6c3bb" +checksum = "c5c9558d4d4f64d006196dd05e01bef3ac25e4250164f04e89f6461b8d8130f8" dependencies = [ "deltalake-aws", "deltalake-azure", @@ -2726,9 +3580,9 @@ dependencies = [ [[package]] name = "deltalake-aws" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e40e385e5e1403c41f0956ab189d44a8c084e93990fe29af4d396e7ed3cd13f" +checksum = "8e80ccc8edaad2ffd8eaa04732ae9b573cbf88a2ce58f087479427bec718c7e2" dependencies = [ "async-trait", "aws-config", @@ -2742,9 +3596,9 @@ dependencies = [ "deltalake-core", "futures", "maplit", - "object_store", + "object_store 0.12.2", "regex 1.11.1", - "thiserror 2.0.12", + "thiserror", "tokio", "tracing", "url", @@ -2753,17 +3607,17 @@ dependencies = [ [[package]] name = "deltalake-azure" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfbea4786321ebd88e083ec74ce513ec7fcad9ddc880b611770dee012652567" +checksum = "8d79b37806a7e6bb0dfa2a156ddd62e935d4c0cba6f96a2982da5dfe109b0918" dependencies = [ "async-trait", "bytes", "deltalake-core", "futures", - "object_store", + "object_store 0.12.2", "regex 1.11.1", - "thiserror 2.0.12", + "thiserror", "tokio", "tracing", "url", @@ -2771,51 +3625,49 @@ dependencies = [ [[package]] name = "deltalake-core" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb0e2d408fe4cb2c3a81c241c8128fdd359dca92a74367b8671fbac206483163" -dependencies = [ - "arrow", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a93bdeb209b17fac1733beea819810689c0b7e88be3d580f9f548d9da6168c" +dependencies = [ + "arrow 55.0.0", + "arrow-arith 55.0.0", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", + "arrow-ipc 55.0.0", + "arrow-json 55.0.0", + "arrow-ord 55.0.0", + "arrow-row 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", "async-trait", "bytes", "cfg-if", "chrono", "dashmap", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-physical-expr", - "datafusion-physical-plan", + "datafusion 47.0.0", + "datafusion-common 47.0.0", + "datafusion-expr 47.0.0", + "datafusion-functions 47.0.0", + "datafusion-functions-aggregate 47.0.0", + "datafusion-physical-expr 47.0.0", + "datafusion-physical-plan 47.0.0", "datafusion-proto", - "datafusion-sql", - "delta_kernel", + "datafusion-sql 47.0.0", + "delta_kernel 0.10.0", + "deltalake-derive", "either", - "errno", - "fix-hidden-lifetime-bug", "futures", "humantime", "indexmap 2.9.0", "itertools 0.14.0", - "libc", "maplit", "num-bigint", "num-traits", "num_cpus", - "object_store", + "object_store 0.12.2", "parking_lot 0.12.3", - "parquet", + "parquet 55.0.0", "percent-encoding", "pin-project-lite", "rand 0.8.5", @@ -2823,9 +3675,9 @@ dependencies = [ "roaring", "serde", "serde_json", - "sqlparser 0.53.0", + "sqlparser 0.56.0", "strum", - "thiserror 2.0.12", + "thiserror", "tokio", "tracing", "url", @@ -2834,19 +3686,32 @@ dependencies = [ "z85", ] +[[package]] +name = "deltalake-derive" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdd39efa077173455fa69c17437141d14ec6273a371d7d3d25ea7f30f61d4c9" +dependencies = [ + "convert_case 0.8.0", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "deltalake-gcp" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa80de5b3e9e53eb9a98d976ac1d64a70b798a73d63cdd83497cc317a5063602" +checksum = "68535c5eb131ceeb713bfc7664d12b270ade5631257ad1d3e13663e4143e8d99" dependencies = [ "async-trait", "bytes", "deltalake-core", "futures", - "object_store", + "object_store 0.12.2", "regex 1.11.1", - "thiserror 2.0.12", + "thiserror", "tokio", "tracing", "url", @@ -2889,7 +3754,7 @@ version = "0.99.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", @@ -3145,26 +4010,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "fix-hidden-lifetime-bug" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7b4994e93dd63050356bdde7d417591d1b348523638dc1c1f539f16e338d55" -dependencies = [ - "fix-hidden-lifetime-bug-proc_macros", -] - -[[package]] -name = "fix-hidden-lifetime-bug-proc_macros" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8f0de9daf465d763422866d0538f07be1596e05623e120b37b4f715f5585200" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "fixedbitset" version = "0.5.7" @@ -3181,6 +4026,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flatbuffers" +version = "25.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" +dependencies = [ + "bitflags 2.9.0", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.1" @@ -3188,6 +4043,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide 0.8.8", ] @@ -3274,7 +4130,7 @@ dependencies = [ "parking_lot 0.12.3", "pin-project", "serde", - "thiserror 2.0.12", + "thiserror", "tokio", ] @@ -3307,7 +4163,7 @@ dependencies = [ "parking_lot 0.12.3", "pin-project", "serde", - "thiserror 2.0.12", + "thiserror", "tokio", "tracing", ] @@ -3342,7 +4198,7 @@ dependencies = [ "pin-project", "rand 0.9.0", "serde", - "thiserror 2.0.12", + "thiserror", "tokio", "tracing", "twox-hash 2.1.0", @@ -4369,6 +5225,15 @@ dependencies = [ "escape8259", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -4543,10 +5408,10 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3641f6a55539a8b6e5349b3bdfb5b315714fbceda3253815838f49e40e3ea757" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", "bytemuck", "half", "serde", @@ -4858,6 +5723,44 @@ dependencies = [ "walkdir", ] +[[package]] +name = "object_store" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781f96d79ed0f961a7021424ab01840efbda64ae7a505aaea195efc91eaaec4" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http 1.3.1", + "http-body-util", + "httparse", + "humantime", + "hyper 1.6.0", + "itertools 0.14.0", + "md-5", + "parking_lot 0.12.3", + "percent-encoding", + "quick-xml", + "rand 0.9.0", + "reqwest", + "ring", + "rustls-pemfile 2.2.0", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -4924,7 +5827,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.12", + "thiserror", "tracing", ] @@ -4957,7 +5860,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.12", + "thiserror", "tracing", ] @@ -4988,7 +5891,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "serde_json", - "thiserror 2.0.12", + "thiserror", "tracing", ] @@ -5107,18 +6010,18 @@ dependencies = [ [[package]] name = "parquet" -version = "54.2.1" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88838dca3b84d41444a0341b19f347e8098a3898b0f21536654b8b799e11abd" +checksum = "bfb15796ac6f56b429fd99e33ba133783ad75b27c36b4b5ce06f1f82cc97754e" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.3.1", + "arrow-data 54.3.1", + "arrow-ipc 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", "base64 0.22.1", "brotli", "bytes", @@ -5130,7 +6033,7 @@ dependencies = [ "lz4_flex", "num", "num-bigint", - "object_store", + "object_store 0.11.2", "paste", "seq-macro", "simdutf8", @@ -5139,7 +6042,42 @@ dependencies = [ "tokio", "twox-hash 1.6.3", "zstd", - "zstd-sys", +] + +[[package]] +name = "parquet" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd31a8290ac5b19f09ad77ee7a1e6a541f1be7674ad410547d5f1eef6eef4a9c" +dependencies = [ + "ahash 0.8.11", + "arrow-array 55.0.0", + "arrow-buffer 55.0.0", + "arrow-cast 55.0.0", + "arrow-data 55.0.0", + "arrow-ipc 55.0.0", + "arrow-schema 55.0.0", + "arrow-select 55.0.0", + "base64 0.22.1", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.15.2", + "lz4_flex", + "num", + "num-bigint", + "object_store 0.12.2", + "paste", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash 2.1.0", + "zstd", ] [[package]] @@ -5191,7 +6129,7 @@ dependencies = [ "postgres-types", "rand 0.8.5", "rust_decimal", - "thiserror 2.0.12", + "thiserror", "tokio", "tokio-rustls 0.26.2", "tokio-util", @@ -5540,7 +6478,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls 0.23.26", "socket2", - "thiserror 2.0.12", + "thiserror", "tokio", "tracing", "web-time", @@ -5560,7 +6498,7 @@ dependencies = [ "rustls 0.23.26", "rustls-pki-types", "slab", - "thiserror 2.0.12", + "thiserror", "tinyvec", "tracing", "web-time", @@ -6240,8 +7178,8 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0462b8e06478cd310e8de11ea2e64c214522275a0b537b3879dbed24a9e01b5" dependencies = [ - "arrow-array", - "arrow-schema", + "arrow-array 54.3.1", + "arrow-schema 54.3.1", "bytemuck", "chrono", "half", @@ -6529,24 +7467,26 @@ dependencies = [ "similar", "subst", "tempfile", - "thiserror 2.0.12", + "thiserror", "tracing", ] [[package]] name = "sqlparser" -version = "0.53.0" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8" +checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899" dependencies = [ "log 0.4.27", + "recursive", + "sqlparser_derive", ] [[package]] name = "sqlparser" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899" +checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" dependencies = [ "log 0.4.27", "recursive", @@ -6555,9 +7495,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.55.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" +checksum = "e68feb51ffa54fc841e086f58da543facfe3d7ae2a60d69b0a8cbbd30d16ae8d" dependencies = [ "log 0.4.27", "recursive", @@ -6624,18 +7564,18 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ "heck", "proc-macro2", @@ -6760,33 +7700,13 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - [[package]] name = "thiserror" version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.12", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", + "thiserror-impl", ] [[package]] @@ -6889,8 +7809,8 @@ dependencies = [ "actix-service", "actix-web", "anyhow", - "arrow", - "arrow-schema", + "arrow 54.2.1", + "arrow-schema 54.3.1", "async-stream", "async-trait", "aws-config", @@ -6903,12 +7823,12 @@ dependencies = [ "color-eyre", "criterion", "crossbeam", - "datafusion", - "datafusion-common", + "datafusion 46.0.1", + "datafusion-common 46.0.1", "datafusion-functions-json", "datafusion-postgres", "datafusion-uwheel", - "delta_kernel", + "delta_kernel 0.12.1", "deltalake", "dotenv", "env_logger", @@ -6918,7 +7838,7 @@ dependencies = [ "lazy_static", "log 0.4.27", "metrics", - "object_store", + "object_store 0.11.2", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", @@ -7493,17 +8413,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "visibility" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "vsimd" version = "0.8.0" @@ -8112,6 +9021,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "zlib-rs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 8fd11767..4913951a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,11 +21,11 @@ color-eyre = "0.6.3" arrow-schema = "54.1.0" regex = "1.11.1" deltalake = { version = "0.26.2", features = ["datafusion", "s3","azure", "gcs",] } -delta_kernel = { version = "0.8.0", features = [ +delta_kernel = { version = "0.12.1", features = [ "arrow-conversion", "default-engine", ] } -chrono = { version = "0.4.39", features = ["serde"] } +chrono = { version = "0.4.41", features = ["serde"] } pgwire = "0.28.0" futures = "0.3.31" bytes = "1.4" From c327111c04ee1add9d528791e288885dede568a9 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 16 Jun 2025 12:23:40 +0100 Subject: [PATCH 28/33] fix : arrow feature issues --- Cargo.lock | 3 +++ Cargo.toml | 1 + 2 files changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 6dc70a2e..ed33f6e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3524,12 +3524,15 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de0b553d03fce69da6bedd91ec7e2348d52af2783a95d2dc91970df0cb614783" dependencies = [ + "arrow 55.0.0", "bytes", "chrono", "delta_kernel_derive 0.12.1", "futures", "indexmap 2.9.0", "itertools 0.14.0", + "object_store 0.12.2", + "parquet 55.0.0", "reqwest", "roaring", "rustc_version", diff --git a/Cargo.toml b/Cargo.toml index 4913951a..e2bca67a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ deltalake = { version = "0.26.2", features = ["datafusion", "s3","azure", "gcs", delta_kernel = { version = "0.12.1", features = [ "arrow-conversion", "default-engine", + "arrow-55" ] } chrono = { version = "0.4.41", features = ["serde"] } pgwire = "0.28.0" From b10f1b4ac9796674133350f8830a1517297a14d6 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 16 Jun 2025 13:50:04 +0100 Subject: [PATCH 29/33] fix : revert version changes --- Cargo.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e2bca67a..8fd11767 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,12 +21,11 @@ color-eyre = "0.6.3" arrow-schema = "54.1.0" regex = "1.11.1" deltalake = { version = "0.26.2", features = ["datafusion", "s3","azure", "gcs",] } -delta_kernel = { version = "0.12.1", features = [ +delta_kernel = { version = "0.8.0", features = [ "arrow-conversion", "default-engine", - "arrow-55" ] } -chrono = { version = "0.4.41", features = ["serde"] } +chrono = { version = "0.4.39", features = ["serde"] } pgwire = "0.28.0" futures = "0.3.31" bytes = "1.4" From d45569d9ada50db4cd2bee7f566cd2c8289ddad6 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 16 Jun 2025 14:05:34 +0100 Subject: [PATCH 30/33] fix : revert version changes --- Cargo.lock | 1710 ++++++++++++---------------------------------------- Cargo.toml | 2 +- 2 files changed, 397 insertions(+), 1315 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed33f6e7..1414e9f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,97 +409,45 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc208515aa0151028e464cc94a692156e945ce5126abd3537bb7fd6ba2143ed1" dependencies = [ - "arrow-arith 54.3.1", - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-cast 54.3.1", - "arrow-csv 54.3.1", - "arrow-data 54.3.1", - "arrow-ipc 54.3.1", - "arrow-json 54.3.1", - "arrow-ord 54.2.1", - "arrow-row 54.2.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", - "arrow-string 54.2.1", -] - -[[package]] -name = "arrow" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3095aaf545942ff5abd46654534f15b03a90fba78299d661e045e5d587222f0d" -dependencies = [ - "arrow-arith 55.0.0", - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-cast 55.0.0", - "arrow-csv 55.0.0", - "arrow-data 55.0.0", - "arrow-ipc 55.0.0", - "arrow-json 55.0.0", - "arrow-ord 55.0.0", - "arrow-row 55.0.0", - "arrow-schema 55.0.0", - "arrow-select 55.0.0", - "arrow-string 55.0.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] name = "arrow-arith" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "chrono", - "num", -] - -[[package]] -name = "arrow-arith" -version = "55.0.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00752064ff47cee746e816ddb8450520c3a52cbad1e256f6fa861a35f86c45e7" +checksum = "e07e726e2b3f7816a85c6a45b6ec118eeeabf0b2a8c208122ad949437181f49a" dependencies = [ - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-data 55.0.0", - "arrow-schema 55.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num", ] [[package]] name = "arrow-array" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" -dependencies = [ - "ahash 0.8.11", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "chrono", - "chrono-tz", - "half", - "hashbrown 0.15.2", - "num", -] - -[[package]] -name = "arrow-array" -version = "55.0.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cebfe926794fbc1f49ddd0cdaf898956ca9f6e79541efce62dabccfd81380472" +checksum = "a2262eba4f16c78496adfd559a29fe4b24df6088efc9985a873d58e92be022d5" dependencies = [ "ahash 0.8.11", - "arrow-buffer 55.0.0", - "arrow-data 55.0.0", - "arrow-schema 55.0.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", @@ -518,49 +466,17 @@ dependencies = [ "num", ] -[[package]] -name = "arrow-buffer" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0303c7ec4cf1a2c60310fc4d6bbc3350cd051a17bf9e9c0a8e47b4db79277824" -dependencies = [ - "bytes", - "half", - "num", -] - -[[package]] -name = "arrow-cast" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", - "atoi", - "base64 0.22.1", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num", - "ryu", -] - [[package]] name = "arrow-cast" -version = "55.0.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335f769c5a218ea823d3760a743feba1ef7857cba114c01399a891c2fff34285" +checksum = "4103d88c5b441525ed4ac23153be7458494c2b0c9a11115848fdb9b81f6f886a" dependencies = [ - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-data 55.0.0", - "arrow-schema 55.0.0", - "arrow-select 55.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "atoi", "base64 0.22.1", "chrono", @@ -573,29 +489,13 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1644877d8bc9a0ef022d9153dc29375c2bda244c39aec05a91d0e87ccf77995f" -dependencies = [ - "arrow-array 54.3.1", - "arrow-cast 54.3.1", - "arrow-schema 54.3.1", - "chrono", - "csv", - "csv-core", - "lazy_static", - "regex 1.11.1", -] - -[[package]] -name = "arrow-csv" -version = "55.0.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "510db7dfbb4d5761826516cc611d97b3a68835d0ece95b034a052601109c0b1b" +checksum = "43d3cb0914486a3cae19a5cad2598e44e225d53157926d0ada03c20521191a65" dependencies = [ - "arrow-array 55.0.0", - "arrow-cast 55.0.0", - "arrow-schema 55.0.0", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "csv", "csv-core", @@ -609,94 +509,44 @@ version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" dependencies = [ - "arrow-buffer 54.3.1", - "arrow-schema 54.3.1", - "half", - "num", -] - -[[package]] -name = "arrow-data" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8affacf3351a24039ea24adab06f316ded523b6f8c3dbe28fbac5f18743451b" -dependencies = [ - "arrow-buffer 55.0.0", - "arrow-schema 55.0.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] [[package]] name = "arrow-ipc" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ff528658b521e33905334723b795ee56b393dbe9cf76c8b1f64b648c65a60c" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "flatbuffers 24.12.23", - "lz4_flex", -] - -[[package]] -name = "arrow-ipc" -version = "55.0.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69880a9e6934d9cba2b8630dd08a3463a91db8693b16b499d54026b6137af284" +checksum = "ddecdeab02491b1ce88885986e25002a3da34dd349f682c7cfe67bab7cc17b86" dependencies = [ - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-data 55.0.0", - "arrow-schema 55.0.0", - "flatbuffers 25.2.10", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "flatbuffers", "lz4_flex", ] [[package]] name = "arrow-json" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee5b4ca98a7fb2efb9ab3309a5d1c88b5116997ff93f3147efdc1062a6158e9" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-cast 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "chrono", - "half", - "indexmap 2.9.0", - "lexical-core", - "memchr", - "num", - "serde", - "serde_json", - "simdutf8", -] - -[[package]] -name = "arrow-json" -version = "55.0.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8dafd17a05449e31e0114d740530e0ada7379d7cb9c338fd65b09a8130960b0" +checksum = "d03b9340013413eb84868682ace00a1098c81a5ebc96d279f7ebf9a4cac3c0fd" dependencies = [ - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-cast 55.0.0", - "arrow-data 55.0.0", - "arrow-schema 55.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap 2.9.0", "lexical-core", - "memchr", "num", "serde", "serde_json", - "simdutf8", ] [[package]] @@ -705,24 +555,11 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f841bfcc1997ef6ac48ee0305c4dfceb1f7c786fe31e67c1186edf775e1f1160" dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", -] - -[[package]] -name = "arrow-ord" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895644523af4e17502d42c3cb6b27cb820f0cb77954c22d75c23a85247c849e1" -dependencies = [ - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-data 55.0.0", - "arrow-schema 55.0.0", - "arrow-select 55.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] @@ -731,23 +568,10 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eeb55b0a0a83851aa01f2ca5ee5648f607e8506ba6802577afdda9d75cdedcd" dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "half", -] - -[[package]] -name = "arrow-row" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be8a2a4e5e7d9c822b2b8095ecd77010576d824f654d347817640acfc97d229" -dependencies = [ - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-data 55.0.0", - "arrow-schema 55.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] @@ -756,12 +580,6 @@ name = "arrow-schema" version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" - -[[package]] -name = "arrow-schema" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49" dependencies = [ "bitflags 2.9.0", "serde", @@ -769,29 +587,15 @@ dependencies = [ [[package]] name = "arrow-select" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" -dependencies = [ - "ahash 0.8.11", - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "num", -] - -[[package]] -name = "arrow-select" -version = "55.0.0" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa5f5a93c75f46ef48e4001535e7b6c922eeb0aa20b73cf58d09e13d057490d8" +checksum = "7e2932aece2d0c869dd2125feb9bd1709ef5c445daa3838ac4112dcfa0fda52c" dependencies = [ "ahash 0.8.11", - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-data 55.0.0", - "arrow-schema 55.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] @@ -801,28 +605,11 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "912e38bd6a7a7714c1d9b61df80315685553b7455e8a6045c27531d8ecd5b458" dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", - "memchr", - "num", - "regex 1.11.1", - "regex-syntax 0.8.5", -] - -[[package]] -name = "arrow-string" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7005d858d84b56428ba2a98a107fe88c0132c61793cf6b8232a1f9bfc0452b" -dependencies = [ - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-data 55.0.0", - "arrow-schema 55.0.0", - "arrow-select 55.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num", "regex 1.11.1", @@ -1766,9 +1553,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1776,7 +1563,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-targets 0.52.6", ] [[package]] @@ -1996,15 +1783,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" -[[package]] -name = "convert_case" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "cookie" version = "0.16.2" @@ -2356,40 +2134,40 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "914e6f9525599579abbd90b0f7a55afcaaaa40350b9e9ed52563f126dfe45fd3" dependencies = [ - "arrow 54.2.1", - "arrow-ipc 54.3.1", - "arrow-schema 54.3.1", + "arrow", + "arrow-ipc", + "arrow-schema", "async-trait", "bytes", "bzip2", "chrono", - "datafusion-catalog 46.0.1", - "datafusion-catalog-listing 46.0.1", - "datafusion-common 46.0.1", - "datafusion-common-runtime 46.0.1", - "datafusion-datasource 46.0.1", - "datafusion-execution 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-expr-common 46.0.1", - "datafusion-functions 46.0.1", - "datafusion-functions-aggregate 46.0.1", - "datafusion-functions-nested 46.0.1", - "datafusion-functions-table 46.0.1", - "datafusion-functions-window 46.0.1", - "datafusion-macros 46.0.1", - "datafusion-optimizer 46.0.1", - "datafusion-physical-expr 46.0.1", - "datafusion-physical-expr-common 46.0.1", - "datafusion-physical-optimizer 46.0.1", - "datafusion-physical-plan 46.0.1", - "datafusion-sql 46.0.1", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-macros", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-sql", "flate2", "futures", "itertools 0.14.0", "log 0.4.27", - "object_store 0.11.2", + "object_store", "parking_lot 0.12.3", - "parquet 54.3.1", + "parquet", "rand 0.8.5", "regex 1.11.1", "sqlparser 0.54.0", @@ -2401,149 +2179,45 @@ dependencies = [ "zstd", ] -[[package]] -name = "datafusion" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" -dependencies = [ - "arrow 55.0.0", - "arrow-ipc 55.0.0", - "arrow-schema 55.0.0", - "async-trait", - "bytes", - "bzip2", - "chrono", - "datafusion-catalog 47.0.0", - "datafusion-catalog-listing 47.0.0", - "datafusion-common 47.0.0", - "datafusion-common-runtime 47.0.0", - "datafusion-datasource 47.0.0", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-expr-common 47.0.0", - "datafusion-functions 47.0.0", - "datafusion-functions-aggregate 47.0.0", - "datafusion-functions-nested 47.0.0", - "datafusion-functions-table 47.0.0", - "datafusion-functions-window 47.0.0", - "datafusion-macros 47.0.0", - "datafusion-optimizer 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", - "datafusion-physical-optimizer 47.0.0", - "datafusion-physical-plan 47.0.0", - "datafusion-session", - "datafusion-sql 47.0.0", - "flate2", - "futures", - "itertools 0.14.0", - "log 0.4.27", - "object_store 0.12.2", - "parking_lot 0.12.3", - "parquet 55.0.0", - "rand 0.8.5", - "regex 1.11.1", - "sqlparser 0.55.0", - "tempfile", - "tokio", - "url", - "uuid", - "xz2", - "zstd", -] - [[package]] name = "datafusion-catalog" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "998a6549e6ee4ee3980e05590b2960446a56b343ea30199ef38acd0e0b9036e2" dependencies = [ - "arrow 54.2.1", + "arrow", "async-trait", "dashmap", - "datafusion-common 46.0.1", - "datafusion-execution 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-physical-plan 46.0.1", - "datafusion-sql 46.0.1", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "datafusion-sql", "futures", "itertools 0.14.0", "log 0.4.27", "parking_lot 0.12.3", ] -[[package]] -name = "datafusion-catalog" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" -dependencies = [ - "arrow 55.0.0", - "async-trait", - "dashmap", - "datafusion-common 47.0.0", - "datafusion-common-runtime 47.0.0", - "datafusion-datasource 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-plan 47.0.0", - "datafusion-session", - "datafusion-sql 47.0.0", - "futures", - "itertools 0.14.0", - "log 0.4.27", - "object_store 0.12.2", - "parking_lot 0.12.3", - "tokio", -] - [[package]] name = "datafusion-catalog-listing" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5ac10096a5b3c0d8a227176c0e543606860842e943594ccddb45cf42a526e43" dependencies = [ - "arrow 54.2.1", + "arrow", "async-trait", - "datafusion-catalog 46.0.1", - "datafusion-common 46.0.1", - "datafusion-datasource 46.0.1", - "datafusion-execution 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-physical-expr 46.0.1", - "datafusion-physical-expr-common 46.0.1", - "datafusion-physical-plan 46.0.1", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "futures", "log 0.4.27", - "object_store 0.11.2", - "tokio", -] - -[[package]] -name = "datafusion-catalog-listing" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" -dependencies = [ - "arrow 55.0.0", - "async-trait", - "datafusion-catalog 47.0.0", - "datafusion-common 47.0.0", - "datafusion-datasource 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", - "datafusion-physical-plan 47.0.0", - "datafusion-session", - "futures", - "log 0.4.27", - "object_store 0.12.2", + "object_store", "tokio", ] @@ -2554,16 +2228,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f53d7ec508e1b3f68bd301cee3f649834fad51eff9240d898a4b2614cfd0a7a" dependencies = [ "ahash 0.8.11", - "arrow 54.2.1", - "arrow-ipc 54.3.1", + "arrow", + "arrow-ipc", "base64 0.22.1", "half", "hashbrown 0.14.5", "indexmap 2.9.0", "libc", "log 0.4.27", - "object_store 0.11.2", - "parquet 54.3.1", + "object_store", + "parquet", "paste", "recursive", "sqlparser 0.54.0", @@ -2571,30 +2245,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "datafusion-common" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" -dependencies = [ - "ahash 0.8.11", - "arrow 55.0.0", - "arrow-ipc 55.0.0", - "base64 0.22.1", - "half", - "hashbrown 0.14.5", - "indexmap 2.9.0", - "libc", - "log 0.4.27", - "object_store 0.12.2", - "parquet 55.0.0", - "paste", - "recursive", - "sqlparser 0.55.0", - "tokio", - "web-time", -] - [[package]] name = "datafusion-common-runtime" version = "46.0.1" @@ -2605,43 +2255,32 @@ dependencies = [ "tokio", ] -[[package]] -name = "datafusion-common-runtime" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5167bb1d2ccbb87c6bc36c295274d7a0519b14afcfdaf401d53cbcaa4ef4968b" -dependencies = [ - "futures", - "log 0.4.27", - "tokio", -] - [[package]] name = "datafusion-datasource" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf7f37ad8b6e88b46c7eeab3236147d32ea64b823544f498455a8d9042839c92" dependencies = [ - "arrow 54.2.1", + "arrow", "async-compression", "async-trait", "bytes", "bzip2", "chrono", - "datafusion-catalog 46.0.1", - "datafusion-common 46.0.1", - "datafusion-common-runtime 46.0.1", - "datafusion-execution 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-physical-expr 46.0.1", - "datafusion-physical-expr-common 46.0.1", - "datafusion-physical-plan 46.0.1", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "flate2", "futures", "glob", "itertools 0.14.0", "log 0.4.27", - "object_store 0.11.2", + "object_store", "rand 0.8.5", "tokio", "tokio-util", @@ -2650,167 +2289,25 @@ dependencies = [ "zstd", ] -[[package]] -name = "datafusion-datasource" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" -dependencies = [ - "arrow 55.0.0", - "async-compression", - "async-trait", - "bytes", - "bzip2", - "chrono", - "datafusion-common 47.0.0", - "datafusion-common-runtime 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", - "datafusion-physical-plan 47.0.0", - "datafusion-session", - "flate2", - "futures", - "glob", - "itertools 0.14.0", - "log 0.4.27", - "object_store 0.12.2", - "parquet 55.0.0", - "rand 0.8.5", - "tempfile", - "tokio", - "tokio-util", - "url", - "xz2", - "zstd", -] - -[[package]] -name = "datafusion-datasource-csv" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" -dependencies = [ - "arrow 55.0.0", - "async-trait", - "bytes", - "datafusion-catalog 47.0.0", - "datafusion-common 47.0.0", - "datafusion-common-runtime 47.0.0", - "datafusion-datasource 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", - "datafusion-physical-plan 47.0.0", - "datafusion-session", - "futures", - "object_store 0.12.2", - "regex 1.11.1", - "tokio", -] - -[[package]] -name = "datafusion-datasource-json" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" -dependencies = [ - "arrow 55.0.0", - "async-trait", - "bytes", - "datafusion-catalog 47.0.0", - "datafusion-common 47.0.0", - "datafusion-common-runtime 47.0.0", - "datafusion-datasource 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", - "datafusion-physical-plan 47.0.0", - "datafusion-session", - "futures", - "object_store 0.12.2", - "serde_json", - "tokio", -] - -[[package]] -name = "datafusion-datasource-parquet" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" -dependencies = [ - "arrow 55.0.0", - "async-trait", - "bytes", - "datafusion-catalog 47.0.0", - "datafusion-common 47.0.0", - "datafusion-common-runtime 47.0.0", - "datafusion-datasource 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-functions-aggregate 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", - "datafusion-physical-optimizer 47.0.0", - "datafusion-physical-plan 47.0.0", - "datafusion-session", - "futures", - "itertools 0.14.0", - "log 0.4.27", - "object_store 0.12.2", - "parking_lot 0.12.3", - "parquet 55.0.0", - "rand 0.8.5", - "tokio", -] - [[package]] name = "datafusion-doc" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7db7a0239fd060f359dc56c6e7db726abaa92babaed2fb2e91c3a8b2fff8b256" -[[package]] -name = "datafusion-doc" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91f8c2c5788ef32f48ff56c68e5b545527b744822a284373ac79bba1ba47292" - [[package]] name = "datafusion-execution" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0938f9e5b6bc5782be4111cdfb70c02b7b5451bf34fd57e4de062a7f7c4e31f1" dependencies = [ - "arrow 54.2.1", + "arrow", "dashmap", - "datafusion-common 46.0.1", - "datafusion-expr 46.0.1", + "datafusion-common", + "datafusion-expr", "futures", "log 0.4.27", - "object_store 0.11.2", - "parking_lot 0.12.3", - "rand 0.8.5", - "tempfile", - "url", -] - -[[package]] -name = "datafusion-execution" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" -dependencies = [ - "arrow 55.0.0", - "dashmap", - "datafusion-common 47.0.0", - "datafusion-expr 47.0.0", - "futures", - "log 0.4.27", - "object_store 0.12.2", + "object_store", "parking_lot 0.12.3", "rand 0.8.5", "tempfile", @@ -2823,14 +2320,14 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b36c28b00b00019a8695ad7f1a53ee1673487b90322ecbd604e2cf32894eb14f" dependencies = [ - "arrow 54.2.1", + "arrow", "chrono", - "datafusion-common 46.0.1", - "datafusion-doc 46.0.1", - "datafusion-expr-common 46.0.1", - "datafusion-functions-aggregate-common 46.0.1", - "datafusion-functions-window-common 46.0.1", - "datafusion-physical-expr-common 46.0.1", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", "indexmap 2.9.0", "paste", "recursive", @@ -2838,48 +2335,14 @@ dependencies = [ "sqlparser 0.54.0", ] -[[package]] -name = "datafusion-expr" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" -dependencies = [ - "arrow 55.0.0", - "chrono", - "datafusion-common 47.0.0", - "datafusion-doc 47.0.0", - "datafusion-expr-common 47.0.0", - "datafusion-functions-aggregate-common 47.0.0", - "datafusion-functions-window-common 47.0.0", - "datafusion-physical-expr-common 47.0.0", - "indexmap 2.9.0", - "paste", - "recursive", - "serde_json", - "sqlparser 0.55.0", -] - [[package]] name = "datafusion-expr-common" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e" dependencies = [ - "arrow 54.2.1", - "datafusion-common 46.0.1", - "indexmap 2.9.0", - "itertools 0.14.0", - "paste", -] - -[[package]] -name = "datafusion-expr-common" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" -dependencies = [ - "arrow 55.0.0", - "datafusion-common 47.0.0", + "arrow", + "datafusion-common", "indexmap 2.9.0", "itertools 0.14.0", "paste", @@ -2891,47 +2354,18 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3196e37d7b65469fb79fee4f05e5bb58a456831035f9a38aa5919aeb3298d40" dependencies = [ - "arrow 54.2.1", - "arrow-buffer 54.3.1", + "arrow", + "arrow-buffer", "base64 0.22.1", "blake2", "blake3", "chrono", - "datafusion-common 46.0.1", - "datafusion-doc 46.0.1", - "datafusion-execution 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-expr-common 46.0.1", - "datafusion-macros 46.0.1", - "hex", - "itertools 0.14.0", - "log 0.4.27", - "md-5", - "rand 0.8.5", - "regex 1.11.1", - "sha2", - "unicode-segmentation", - "uuid", -] - -[[package]] -name = "datafusion-functions" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" -dependencies = [ - "arrow 55.0.0", - "arrow-buffer 55.0.0", - "base64 0.22.1", - "blake2", - "blake3", - "chrono", - "datafusion-common 47.0.0", - "datafusion-doc 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-expr-common 47.0.0", - "datafusion-macros 47.0.0", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", "hex", "itertools 0.14.0", "log 0.4.27", @@ -2950,36 +2384,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adfc2d074d5ee4d9354fdcc9283d5b2b9037849237ddecb8942a29144b77ca05" dependencies = [ "ahash 0.8.11", - "arrow 54.2.1", - "datafusion-common 46.0.1", - "datafusion-doc 46.0.1", - "datafusion-execution 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-functions-aggregate-common 46.0.1", - "datafusion-macros 46.0.1", - "datafusion-physical-expr 46.0.1", - "datafusion-physical-expr-common 46.0.1", - "half", - "log 0.4.27", - "paste", -] - -[[package]] -name = "datafusion-functions-aggregate" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" -dependencies = [ - "ahash 0.8.11", - "arrow 55.0.0", - "datafusion-common 47.0.0", - "datafusion-doc 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-functions-aggregate-common 47.0.0", - "datafusion-macros 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "half", "log 0.4.27", "paste", @@ -2992,23 +2405,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cbceba0f98d921309a9121b702bcd49289d383684cccabf9a92cda1602f3bbb" dependencies = [ "ahash 0.8.11", - "arrow 54.2.1", - "datafusion-common 46.0.1", - "datafusion-expr-common 46.0.1", - "datafusion-physical-expr-common 46.0.1", -] - -[[package]] -name = "datafusion-functions-aggregate-common" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" -dependencies = [ - "ahash 0.8.11", - "arrow 55.0.0", - "datafusion-common 47.0.0", - "datafusion-expr-common 47.0.0", - "datafusion-physical-expr-common 47.0.0", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", ] [[package]] @@ -3017,7 +2417,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9658d1ad5c3ac21667d04d01222202cb644fd85b2c5ea9d82c4efa33153d90" dependencies = [ - "datafusion 46.0.1", + "datafusion", "jiter", "log 0.4.27", "paste", @@ -3029,37 +2429,16 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "170e27ce4baa27113ddf5f77f1a7ec484b0dbeda0c7abbd4bad3fc609c8ab71a" dependencies = [ - "arrow 54.2.1", - "arrow-ord 54.2.1", - "datafusion-common 46.0.1", - "datafusion-doc 46.0.1", - "datafusion-execution 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-functions 46.0.1", - "datafusion-functions-aggregate 46.0.1", - "datafusion-macros 46.0.1", - "datafusion-physical-expr-common 46.0.1", - "itertools 0.14.0", - "log 0.4.27", - "paste", -] - -[[package]] -name = "datafusion-functions-nested" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" -dependencies = [ - "arrow 55.0.0", - "arrow-ord 55.0.0", - "datafusion-common 47.0.0", - "datafusion-doc 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-functions 47.0.0", - "datafusion-functions-aggregate 47.0.0", - "datafusion-macros 47.0.0", - "datafusion-physical-expr-common 47.0.0", + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-macros", + "datafusion-physical-expr-common", "itertools 0.14.0", "log 0.4.27", "paste", @@ -3071,28 +2450,12 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d3a06a7f0817ded87b026a437e7e51de7f59d48173b0a4e803aa896a7bd6bb5" dependencies = [ - "arrow 54.2.1", - "async-trait", - "datafusion-catalog 46.0.1", - "datafusion-common 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-physical-plan 46.0.1", - "parking_lot 0.12.3", - "paste", -] - -[[package]] -name = "datafusion-functions-table" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" -dependencies = [ - "arrow 55.0.0", + "arrow", "async-trait", - "datafusion-catalog 47.0.0", - "datafusion-common 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-physical-plan 47.0.0", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", "parking_lot 0.12.3", "paste", ] @@ -3103,30 +2466,13 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6c608b66496a1e05e3d196131eb9bebea579eed1f59e88d962baf3dda853bc6" dependencies = [ - "datafusion-common 46.0.1", - "datafusion-doc 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-functions-window-common 46.0.1", - "datafusion-macros 46.0.1", - "datafusion-physical-expr 46.0.1", - "datafusion-physical-expr-common 46.0.1", - "log 0.4.27", - "paste", -] - -[[package]] -name = "datafusion-functions-window" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefc2d77646e1aadd1d6a9c40088937aedec04e68c5f0465939912e1291f8193" -dependencies = [ - "datafusion-common 47.0.0", - "datafusion-doc 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-functions-window-common 47.0.0", - "datafusion-macros 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "log 0.4.27", "paste", ] @@ -3137,18 +2483,8 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da2f9d83348957b4ad0cd87b5cb9445f2651863a36592fe5484d43b49a5f8d82" dependencies = [ - "datafusion-common 46.0.1", - "datafusion-physical-expr-common 46.0.1", -] - -[[package]] -name = "datafusion-functions-window-common" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4aff082c42fa6da99ce0698c85addd5252928c908eb087ca3cfa64ff16b313" -dependencies = [ - "datafusion-common 47.0.0", - "datafusion-physical-expr-common 47.0.0", + "datafusion-common", + "datafusion-physical-expr-common", ] [[package]] @@ -3157,18 +2493,7 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4800e1ff7ecf8f310887e9b54c9c444b8e215ccbc7b21c2f244cfae373b1ece7" dependencies = [ - "datafusion-expr 46.0.1", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "datafusion-macros" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" -dependencies = [ - "datafusion-expr 47.0.0", + "datafusion-expr", "quote", "syn 2.0.100", ] @@ -3179,73 +2504,32 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "971c51c54cd309001376fae752fb15a6b41750b6d1552345c46afbfb6458801b" dependencies = [ - "arrow 54.2.1", - "chrono", - "datafusion-common 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-physical-expr 46.0.1", - "indexmap 2.9.0", - "itertools 0.14.0", - "log 0.4.27", - "recursive", - "regex 1.11.1", - "regex-syntax 0.8.5", -] - -[[package]] -name = "datafusion-optimizer" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" -dependencies = [ - "arrow 55.0.0", + "arrow", "chrono", - "datafusion-common 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-physical-expr 47.0.0", - "indexmap 2.9.0", - "itertools 0.14.0", - "log 0.4.27", - "recursive", - "regex 1.11.1", - "regex-syntax 0.8.5", -] - -[[package]] -name = "datafusion-physical-expr" -version = "46.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85" -dependencies = [ - "ahash 0.8.11", - "arrow 54.2.1", - "datafusion-common 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-expr-common 46.0.1", - "datafusion-functions-aggregate-common 46.0.1", - "datafusion-physical-expr-common 46.0.1", - "half", - "hashbrown 0.14.5", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", "indexmap 2.9.0", "itertools 0.14.0", "log 0.4.27", - "paste", - "petgraph", + "recursive", + "regex 1.11.1", + "regex-syntax 0.8.5", ] [[package]] name = "datafusion-physical-expr" -version = "47.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" +checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85" dependencies = [ "ahash 0.8.11", - "arrow 55.0.0", - "datafusion-common 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-expr-common 47.0.0", - "datafusion-functions-aggregate-common 47.0.0", - "datafusion-physical-expr-common 47.0.0", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", "indexmap 2.9.0", @@ -3262,23 +2546,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f8c25dcd069073a75b3d2840a79d0f81e64bdd2c05f2d3d18939afb36a7dcb" dependencies = [ "ahash 0.8.11", - "arrow 54.2.1", - "datafusion-common 46.0.1", - "datafusion-expr-common 46.0.1", - "hashbrown 0.14.5", - "itertools 0.14.0", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" -dependencies = [ - "ahash 0.8.11", - "arrow 55.0.0", - "datafusion-common 47.0.0", - "datafusion-expr-common 47.0.0", + "arrow", + "datafusion-common", + "datafusion-expr-common", "hashbrown 0.14.5", "itertools 0.14.0", ] @@ -3289,33 +2559,14 @@ version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68da5266b5b9847c11d1b3404ee96b1d423814e1973e1ad3789131e5ec912763" dependencies = [ - "arrow 54.2.1", - "datafusion-common 46.0.1", - "datafusion-execution 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-expr-common 46.0.1", - "datafusion-physical-expr 46.0.1", - "datafusion-physical-expr-common 46.0.1", - "datafusion-physical-plan 46.0.1", - "itertools 0.14.0", - "log 0.4.27", - "recursive", -] - -[[package]] -name = "datafusion-physical-optimizer" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" -dependencies = [ - "arrow 55.0.0", - "datafusion-common 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-expr-common 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", - "datafusion-physical-plan 47.0.0", + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "itertools 0.14.0", "log 0.4.27", "recursive", @@ -3328,48 +2579,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88cc160df00e413e370b3b259c8ea7bfbebc134d32de16325950e9e923846b7f" dependencies = [ "ahash 0.8.11", - "arrow 54.2.1", - "arrow-ord 54.2.1", - "arrow-schema 54.3.1", - "async-trait", - "chrono", - "datafusion-common 46.0.1", - "datafusion-common-runtime 46.0.1", - "datafusion-execution 46.0.1", - "datafusion-expr 46.0.1", - "datafusion-functions-window-common 46.0.1", - "datafusion-physical-expr 46.0.1", - "datafusion-physical-expr-common 46.0.1", - "futures", - "half", - "hashbrown 0.14.5", - "indexmap 2.9.0", - "itertools 0.14.0", - "log 0.4.27", - "parking_lot 0.12.3", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "datafusion-physical-plan" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" -dependencies = [ - "ahash 0.8.11", - "arrow 55.0.0", - "arrow-ord 55.0.0", - "arrow-schema 55.0.0", + "arrow", + "arrow-ord", + "arrow-schema", "async-trait", "chrono", - "datafusion-common 47.0.0", - "datafusion-common-runtime 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-functions-window-common 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-expr-common 47.0.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", "half", "hashbrown 0.14.5", @@ -3388,72 +2609,48 @@ source = "git+https://github.com/sunng87/datafusion-postgres.git?rev=2cf58787a8b dependencies = [ "async-trait", "chrono", - "datafusion 46.0.1", + "datafusion", "futures", "pgwire", ] [[package]] name = "datafusion-proto" -version = "47.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a1afb2bdb05de7ff65be6883ebfd4ec027bd9f1f21c46aa3afd01927160a83" +checksum = "6f6ef4c6eb52370cb48639e25e2331a415aac0b2b0a0a472b36e26603bdf184f" dependencies = [ - "arrow 55.0.0", + "arrow", "chrono", - "datafusion 47.0.0", - "datafusion-common 47.0.0", - "datafusion-expr 47.0.0", + "datafusion", + "datafusion-common", + "datafusion-expr", "datafusion-proto-common", - "object_store 0.12.2", + "object_store", "prost", ] [[package]] name = "datafusion-proto-common" -version = "47.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b7a5876ebd6b564fb9a1fd2c3a2a9686b787071a256b47e4708f0916f9e46f" +checksum = "5faf4a9bbb0d0a305fea8a6db21ba863286b53e53a212e687d2774028dd6f03f" dependencies = [ - "arrow 55.0.0", - "datafusion-common 47.0.0", + "arrow", + "datafusion-common", "prost", ] -[[package]] -name = "datafusion-session" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" -dependencies = [ - "arrow 55.0.0", - "async-trait", - "dashmap", - "datafusion-common 47.0.0", - "datafusion-common-runtime 47.0.0", - "datafusion-execution 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-plan 47.0.0", - "datafusion-sql 47.0.0", - "futures", - "itertools 0.14.0", - "log 0.4.27", - "object_store 0.12.2", - "parking_lot 0.12.3", - "tokio", -] - [[package]] name = "datafusion-sql" version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325a212b67b677c0eb91447bf9a11b630f9fc4f62d8e5d145bf859f5a6b29e64" dependencies = [ - "arrow 54.2.1", + "arrow", "bigdecimal", - "datafusion-common 46.0.1", - "datafusion-expr 46.0.1", + "datafusion-common", + "datafusion-expr", "indexmap 2.9.0", "log 0.4.27", "recursive", @@ -3461,23 +2658,6 @@ dependencies = [ "sqlparser 0.54.0", ] -[[package]] -name = "datafusion-sql" -version = "47.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" -dependencies = [ - "arrow 55.0.0", - "bigdecimal", - "datafusion-common 47.0.0", - "datafusion-expr 47.0.0", - "indexmap 2.9.0", - "log 0.4.27", - "recursive", - "regex 1.11.1", - "sqlparser 0.55.0", -] - [[package]] name = "datafusion-uwheel" version = "46.0.0" @@ -3485,84 +2665,47 @@ source = "git+https://github.com/apitoolkit/datafusion-uwheel.git?branch=datafus dependencies = [ "bitpacking", "chrono", - "datafusion 46.0.1", + "datafusion", "uwheel", ] [[package]] name = "delta_kernel" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96f51383ba327a1403e6e3458f8fc979d09d7200af56fa32681619f6c760dee" -dependencies = [ - "arrow 55.0.0", - "bytes", - "chrono", - "delta_kernel_derive 0.10.0", - "futures", - "indexmap 2.9.0", - "itertools 0.14.0", - "object_store 0.12.2", - "parquet 55.0.0", - "reqwest", - "roaring", - "rustc_version", - "serde", - "serde_json", - "strum", - "thiserror", - "tokio", - "tracing", - "url", - "uuid", - "z85", -] - -[[package]] -name = "delta_kernel" -version = "0.12.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0b553d03fce69da6bedd91ec7e2348d52af2783a95d2dc91970df0cb614783" +checksum = "aae7dc3012ad01882cd7669fd9524d7069cd5a6f12d69932a6f125d3bf503019" dependencies = [ - "arrow 55.0.0", + "arrow", "bytes", "chrono", - "delta_kernel_derive 0.12.1", + "delta_kernel_derive", + "fix-hidden-lifetime-bug", "futures", + "home", "indexmap 2.9.0", - "itertools 0.14.0", - "object_store 0.12.2", - "parquet 55.0.0", + "itertools 0.13.0", + "object_store", + "parquet", "reqwest", "roaring", "rustc_version", "serde", "serde_json", "strum", - "thiserror", + "thiserror 1.0.69", "tokio", "tracing", "url", "uuid", + "visibility", "z85", ] [[package]] name = "delta_kernel_derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b49a2e67ebafbe644e36f251ee985f237bfb39e4ef1e312eb5876535bc449e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "delta_kernel_derive" -version = "0.12.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deacb882456b0a3e7a5bf22a708190758cc8f572e02cd34954931e24286a4509" +checksum = "0c8e41236d5a9f04da3072d7186a76aba734e7bfd2cd05f7877fde172b65fb11" dependencies = [ "proc-macro2", "quote", @@ -3571,9 +2714,9 @@ dependencies = [ [[package]] name = "deltalake" -version = "0.26.2" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5c9558d4d4f64d006196dd05e01bef3ac25e4250164f04e89f6461b8d8130f8" +checksum = "78889f4005974b848f130fa5dedae81987f1bc93b107291ea87d900c93b6c3bb" dependencies = [ "deltalake-aws", "deltalake-azure", @@ -3583,9 +2726,9 @@ dependencies = [ [[package]] name = "deltalake-aws" -version = "0.9.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e80ccc8edaad2ffd8eaa04732ae9b573cbf88a2ce58f087479427bec718c7e2" +checksum = "6e40e385e5e1403c41f0956ab189d44a8c084e93990fe29af4d396e7ed3cd13f" dependencies = [ "async-trait", "aws-config", @@ -3599,9 +2742,9 @@ dependencies = [ "deltalake-core", "futures", "maplit", - "object_store 0.12.2", + "object_store", "regex 1.11.1", - "thiserror", + "thiserror 2.0.12", "tokio", "tracing", "url", @@ -3610,17 +2753,17 @@ dependencies = [ [[package]] name = "deltalake-azure" -version = "0.9.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d79b37806a7e6bb0dfa2a156ddd62e935d4c0cba6f96a2982da5dfe109b0918" +checksum = "3dfbea4786321ebd88e083ec74ce513ec7fcad9ddc880b611770dee012652567" dependencies = [ "async-trait", "bytes", "deltalake-core", "futures", - "object_store 0.12.2", + "object_store", "regex 1.11.1", - "thiserror", + "thiserror 2.0.12", "tokio", "tracing", "url", @@ -3628,49 +2771,51 @@ dependencies = [ [[package]] name = "deltalake-core" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a93bdeb209b17fac1733beea819810689c0b7e88be3d580f9f548d9da6168c" -dependencies = [ - "arrow 55.0.0", - "arrow-arith 55.0.0", - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-cast 55.0.0", - "arrow-ipc 55.0.0", - "arrow-json 55.0.0", - "arrow-ord 55.0.0", - "arrow-row 55.0.0", - "arrow-schema 55.0.0", - "arrow-select 55.0.0", +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb0e2d408fe4cb2c3a81c241c8128fdd359dca92a74367b8671fbac206483163" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", "async-trait", "bytes", "cfg-if", "chrono", "dashmap", - "datafusion 47.0.0", - "datafusion-common 47.0.0", - "datafusion-expr 47.0.0", - "datafusion-functions 47.0.0", - "datafusion-functions-aggregate 47.0.0", - "datafusion-physical-expr 47.0.0", - "datafusion-physical-plan 47.0.0", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-plan", "datafusion-proto", - "datafusion-sql 47.0.0", - "delta_kernel 0.10.0", - "deltalake-derive", + "datafusion-sql", + "delta_kernel", "either", + "errno", + "fix-hidden-lifetime-bug", "futures", "humantime", "indexmap 2.9.0", "itertools 0.14.0", + "libc", "maplit", "num-bigint", "num-traits", "num_cpus", - "object_store 0.12.2", + "object_store", "parking_lot 0.12.3", - "parquet 55.0.0", + "parquet", "percent-encoding", "pin-project-lite", "rand 0.8.5", @@ -3678,9 +2823,9 @@ dependencies = [ "roaring", "serde", "serde_json", - "sqlparser 0.56.0", + "sqlparser 0.53.0", "strum", - "thiserror", + "thiserror 2.0.12", "tokio", "tracing", "url", @@ -3689,32 +2834,19 @@ dependencies = [ "z85", ] -[[package]] -name = "deltalake-derive" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdd39efa077173455fa69c17437141d14ec6273a371d7d3d25ea7f30f61d4c9" -dependencies = [ - "convert_case 0.8.0", - "itertools 0.14.0", - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "deltalake-gcp" -version = "0.10.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68535c5eb131ceeb713bfc7664d12b270ade5631257ad1d3e13663e4143e8d99" +checksum = "fa80de5b3e9e53eb9a98d976ac1d64a70b798a73d63cdd83497cc317a5063602" dependencies = [ "async-trait", "bytes", "deltalake-core", "futures", - "object_store 0.12.2", + "object_store", "regex 1.11.1", - "thiserror", + "thiserror 2.0.12", "tokio", "tracing", "url", @@ -3757,7 +2889,7 @@ version = "0.99.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f" dependencies = [ - "convert_case 0.4.0", + "convert_case", "proc-macro2", "quote", "rustc_version", @@ -4013,6 +3145,26 @@ dependencies = [ "subtle", ] +[[package]] +name = "fix-hidden-lifetime-bug" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7b4994e93dd63050356bdde7d417591d1b348523638dc1c1f539f16e338d55" +dependencies = [ + "fix-hidden-lifetime-bug-proc_macros", +] + +[[package]] +name = "fix-hidden-lifetime-bug-proc_macros" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f0de9daf465d763422866d0538f07be1596e05623e120b37b4f715f5585200" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "fixedbitset" version = "0.5.7" @@ -4029,16 +3181,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "flatbuffers" -version = "25.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" -dependencies = [ - "bitflags 2.9.0", - "rustc_version", -] - [[package]] name = "flate2" version = "1.1.1" @@ -4046,7 +3188,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "libz-rs-sys", "miniz_oxide 0.8.8", ] @@ -4133,7 +3274,7 @@ dependencies = [ "parking_lot 0.12.3", "pin-project", "serde", - "thiserror", + "thiserror 2.0.12", "tokio", ] @@ -4166,7 +3307,7 @@ dependencies = [ "parking_lot 0.12.3", "pin-project", "serde", - "thiserror", + "thiserror 2.0.12", "tokio", "tracing", ] @@ -4201,7 +3342,7 @@ dependencies = [ "pin-project", "rand 0.9.0", "serde", - "thiserror", + "thiserror 2.0.12", "tokio", "tracing", "twox-hash 2.1.0", @@ -5228,15 +4369,6 @@ dependencies = [ "escape8259", ] -[[package]] -name = "libz-rs-sys" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" -dependencies = [ - "zlib-rs", -] - [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -5411,10 +4543,10 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3641f6a55539a8b6e5349b3bdfb5b315714fbceda3253815838f49e40e3ea757" dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "bytemuck", "half", "serde", @@ -5726,44 +4858,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "object_store" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781f96d79ed0f961a7021424ab01840efbda64ae7a505aaea195efc91eaaec4" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes", - "chrono", - "form_urlencoded", - "futures", - "http 1.3.1", - "http-body-util", - "httparse", - "humantime", - "hyper 1.6.0", - "itertools 0.14.0", - "md-5", - "parking_lot 0.12.3", - "percent-encoding", - "quick-xml", - "rand 0.9.0", - "reqwest", - "ring", - "rustls-pemfile 2.2.0", - "serde", - "serde_json", - "serde_urlencoded", - "thiserror", - "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -5830,7 +4924,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror", + "thiserror 2.0.12", "tracing", ] @@ -5863,7 +4957,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror", + "thiserror 2.0.12", "tracing", ] @@ -5894,7 +4988,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "serde_json", - "thiserror", + "thiserror 2.0.12", "tracing", ] @@ -6013,18 +5107,18 @@ dependencies = [ [[package]] name = "parquet" -version = "54.3.1" +version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb15796ac6f56b429fd99e33ba133783ad75b27c36b4b5ce06f1f82cc97754e" +checksum = "f88838dca3b84d41444a0341b19f347e8098a3898b0f21536654b8b799e11abd" dependencies = [ "ahash 0.8.11", - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-cast 54.3.1", - "arrow-data 54.3.1", - "arrow-ipc 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", "base64 0.22.1", "brotli", "bytes", @@ -6036,7 +5130,7 @@ dependencies = [ "lz4_flex", "num", "num-bigint", - "object_store 0.11.2", + "object_store", "paste", "seq-macro", "simdutf8", @@ -6045,42 +5139,7 @@ dependencies = [ "tokio", "twox-hash 1.6.3", "zstd", -] - -[[package]] -name = "parquet" -version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd31a8290ac5b19f09ad77ee7a1e6a541f1be7674ad410547d5f1eef6eef4a9c" -dependencies = [ - "ahash 0.8.11", - "arrow-array 55.0.0", - "arrow-buffer 55.0.0", - "arrow-cast 55.0.0", - "arrow-data 55.0.0", - "arrow-ipc 55.0.0", - "arrow-schema 55.0.0", - "arrow-select 55.0.0", - "base64 0.22.1", - "brotli", - "bytes", - "chrono", - "flate2", - "futures", - "half", - "hashbrown 0.15.2", - "lz4_flex", - "num", - "num-bigint", - "object_store 0.12.2", - "paste", - "seq-macro", - "simdutf8", - "snap", - "thrift", - "tokio", - "twox-hash 2.1.0", - "zstd", + "zstd-sys", ] [[package]] @@ -6132,7 +5191,7 @@ dependencies = [ "postgres-types", "rand 0.8.5", "rust_decimal", - "thiserror", + "thiserror 2.0.12", "tokio", "tokio-rustls 0.26.2", "tokio-util", @@ -6481,7 +5540,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls 0.23.26", "socket2", - "thiserror", + "thiserror 2.0.12", "tokio", "tracing", "web-time", @@ -6501,7 +5560,7 @@ dependencies = [ "rustls 0.23.26", "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.12", "tinyvec", "tracing", "web-time", @@ -7181,8 +6240,8 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0462b8e06478cd310e8de11ea2e64c214522275a0b537b3879dbed24a9e01b5" dependencies = [ - "arrow-array 54.3.1", - "arrow-schema 54.3.1", + "arrow-array", + "arrow-schema", "bytemuck", "chrono", "half", @@ -7470,26 +6529,24 @@ dependencies = [ "similar", "subst", "tempfile", - "thiserror", + "thiserror 2.0.12", "tracing", ] [[package]] name = "sqlparser" -version = "0.54.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899" +checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8" dependencies = [ "log 0.4.27", - "recursive", - "sqlparser_derive", ] [[package]] name = "sqlparser" -version = "0.55.0" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" +checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899" dependencies = [ "log 0.4.27", "recursive", @@ -7498,9 +6555,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.56.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e68feb51ffa54fc841e086f58da543facfe3d7ae2a60d69b0a8cbbd30d16ae8d" +checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" dependencies = [ "log 0.4.27", "recursive", @@ -7567,18 +6624,18 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.27.1" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.27.1" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ "heck", "proc-macro2", @@ -7703,13 +6760,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", ] [[package]] @@ -7812,8 +6889,8 @@ dependencies = [ "actix-service", "actix-web", "anyhow", - "arrow 54.2.1", - "arrow-schema 54.3.1", + "arrow", + "arrow-schema", "async-stream", "async-trait", "aws-config", @@ -7826,12 +6903,12 @@ dependencies = [ "color-eyre", "criterion", "crossbeam", - "datafusion 46.0.1", - "datafusion-common 46.0.1", + "datafusion", + "datafusion-common", "datafusion-functions-json", "datafusion-postgres", "datafusion-uwheel", - "delta_kernel 0.12.1", + "delta_kernel", "deltalake", "dotenv", "env_logger", @@ -7841,7 +6918,7 @@ dependencies = [ "lazy_static", "log 0.4.27", "metrics", - "object_store 0.11.2", + "object_store", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", @@ -8416,6 +7493,17 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "vsimd" version = "0.8.0" @@ -9024,12 +8112,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "zlib-rs" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" - [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 8fd11767..c4073fbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ log = "0.4.25" color-eyre = "0.6.3" arrow-schema = "54.1.0" regex = "1.11.1" -deltalake = { version = "0.26.2", features = ["datafusion", "s3","azure", "gcs",] } +deltalake = { version = "0.25", features = ["datafusion", "s3","azure", "gcs",] } delta_kernel = { version = "0.8.0", features = [ "arrow-conversion", "default-engine", From c8203d3d8882ecd733b2ae5a9318dba11222acf3 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 16 Jun 2025 14:20:49 +0100 Subject: [PATCH 31/33] fix : revert version changes --- src/delta.rs | 3 ++- src/obj_store.rs | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/delta.rs b/src/delta.rs index 9411507e..2791e6a2 100644 --- a/src/delta.rs +++ b/src/delta.rs @@ -4,8 +4,9 @@ use deltalake::{ DeltaTable, DeltaTableBuilder, DeltaTableError, arrow::record_batch::RecordBatch, operations::{create::CreateBuilder, write::WriteBuilder}, - logstore::ObjectStoreRef, + }; +use deltalake_core::storage::ObjectStoreRef; use object_store::{aws::AmazonS3Builder, azure::MicrosoftAzureBuilder, gcp::GoogleCloudStorageBuilder, local::LocalFileSystem, memory::InMemory}; use tokio; use url::Url; diff --git a/src/obj_store.rs b/src/obj_store.rs index d12050af..88271574 100644 --- a/src/obj_store.rs +++ b/src/obj_store.rs @@ -378,11 +378,11 @@ impl DeltaCachedObjectStore { meta: ObjectMeta { location: location.clone(), last_modified: DateTime::::MIN_UTC, - size: cached_obj.original_size as u64, + size: cached_obj.original_size , e_tag: cached_obj.etag.clone(), version: None, }, - range: 0..cached_obj.original_size as u64, + range: 0..cached_obj.original_size , attributes: Default::default(), }); } @@ -402,7 +402,7 @@ impl DeltaCachedObjectStore { let meta = result.meta.clone(); // Only cache if object size is within limits - if meta.size <= self.config.max_object_size as u64 { + if meta.size <= self.config.max_object_size { // Read the entire payload for caching let bytes = result.bytes().await?; @@ -501,7 +501,7 @@ impl ObjectStore for DeltaCachedObjectStore { self.get_with_cache(location, options).await } - async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { + async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { let options = GetOptions { range: Some(GetRange::Bounded(range)), ..Default::default() From 6ad00f293959d9dc8cf79e32674161947477518f Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 16 Jun 2025 15:36:59 +0100 Subject: [PATCH 32/33] fix : revert version changes --- src/delta.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/delta.rs b/src/delta.rs index 2791e6a2..b615f1a6 100644 --- a/src/delta.rs +++ b/src/delta.rs @@ -4,9 +4,9 @@ use deltalake::{ DeltaTable, DeltaTableBuilder, DeltaTableError, arrow::record_batch::RecordBatch, operations::{create::CreateBuilder, write::WriteBuilder}, - + storage::ObjectStoreRef, }; -use deltalake_core::storage::ObjectStoreRef; + use object_store::{aws::AmazonS3Builder, azure::MicrosoftAzureBuilder, gcp::GoogleCloudStorageBuilder, local::LocalFileSystem, memory::InMemory}; use tokio; use url::Url; From 364a46540b0082a46def786cf3471448b770485d Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 29 Jun 2025 00:18:45 +0100 Subject: [PATCH 33/33] extra version changes --- Cargo.toml | 6 +-- src/delta.rs | 124 +++++++++++++++++++++++++---------------------- src/obj_store.rs | 26 ++++++---- 3 files changed, 84 insertions(+), 72 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c4073fbd..c1a5340a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,8 +5,8 @@ edition = "2024" [dependencies] tokio = { version = "1.43", features = ["full"] } -foyer = {version ="0.17.0",features=["serde"]} -object_store = { version = "0.11.2", features = ["aws", "azure", "gcp"] } +foyer = {version ="0.17.3",features=["serde"]} +object_store = { version = "0.12.2", features = ["aws", "azure", "gcp"] } datafusion = "46.0.0" arrow = "54.2.0" uuid = { version = "1.13", features = ["v4", "serde"] } @@ -20,7 +20,7 @@ log = "0.4.25" color-eyre = "0.6.3" arrow-schema = "54.1.0" regex = "1.11.1" -deltalake = { version = "0.25", features = ["datafusion", "s3","azure", "gcs",] } +deltalake = { version = "0.26.2", features = ["datafusion", "s3","azure", "gcs",] } delta_kernel = { version = "0.8.0", features = [ "arrow-conversion", "default-engine", diff --git a/src/delta.rs b/src/delta.rs index b615f1a6..02e57fdf 100644 --- a/src/delta.rs +++ b/src/delta.rs @@ -3,15 +3,16 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use deltalake::{ DeltaTable, DeltaTableBuilder, DeltaTableError, arrow::record_batch::RecordBatch, - operations::{create::CreateBuilder, write::WriteBuilder}, - storage::ObjectStoreRef, + operations::write::WriteBuilder, + logstore::{default_logstore, ObjectStoreRef}, + operations::create::{CreateBuilder}, + kernel::StructField, }; - use object_store::{aws::AmazonS3Builder, azure::MicrosoftAzureBuilder, gcp::GoogleCloudStorageBuilder, local::LocalFileSystem, memory::InMemory}; use tokio; use url::Url; -use crate::obj_store::{CacheMetrics, DeltaCacheBuilder, DeltaCacheConfig}; +use crate::store::{DeltaCacheBuilder, DeltaCacheConfig}; /// Helper struct for creating Delta tables with caching pub struct CachedDeltaTableBuilder { @@ -164,15 +165,15 @@ impl CachedDeltaTableBuilder { } } - - /// Convenience functions for common Delta operations with caching pub struct CachedDeltaOps; impl CachedDeltaOps { /// Create a new Delta table with caching enabled pub async fn create_table( - table_uri: &str, cache_config: Option, + table_uri: &str, + schema: Arc, + cache_config: Option, ) -> Result { let mut builder = CachedDeltaTableBuilder::new(table_uri); @@ -180,15 +181,21 @@ impl CachedDeltaOps { builder = builder.with_cache_config(config); } - let table = builder.build().await?; + let store = builder.create_base_object_store().await?; + let url = Url::parse(table_uri).map_err(|e| DeltaTableError::Generic(e.to_string()))?; + let log_store = default_logstore(store, &url, &Default::default()); + + let columns: Vec = schema + .fields() + .iter() + .map(|f| StructField::try_from(f.as_ref()).unwrap()) + .collect(); - CreateBuilder::new() - .with_log_store(table.log_store()) + CreateBuilder::new() + .with_log_store(log_store) .with_table_name(table_uri) - + .with_columns(columns) .await - - } /// Open an existing Delta table with caching @@ -219,81 +226,54 @@ impl CachedDeltaOps { } } -#[tokio::main] -async fn main() -> Result<(), Box> { - // Example 1: Simple cached Delta table - let cache_config = DeltaCacheConfig { - memory_capacity: 256 * 1024 * 1024, // 256MB - disk_capacity: 1024 * 1024 * 1024, // 1GB - disk_cache_dir: "/tmp/delta_cache".to_string(), - ttl_seconds: 3600, // 1 hour - enable_metrics: true, - enable_cache_warming: true, - ..Default::default() - }; - let table = CachedDeltaTableBuilder::new("s3://my-bucket/my-table") - .with_cache_config(cache_config) - .with_storage_option("AWS_REGION", "us-west-2") - .with_storage_option("AWS_ACCESS_KEY_ID", "your-access-key") - .with_storage_option("AWS_SECRET_ACCESS_KEY", "your-secret-key") - .build() - .await?; - println!("Table loaded with {} files", table.get_files_count()); - - - - Ok(()) -} #[cfg(test)] mod tests { - use std::sync::Arc; + use std::{fmt::Error, sync::Arc}; use arrow::{ array::{Int32Array, StringArray}, datatypes::{DataType, Field, Schema}, + record_batch::RecordBatch, }; - use tempfile::TempDir; + use tempfile::{tempdir, TempDir}; use super::*; #[tokio::test] async fn test_cached_delta_table_creation() { - let temp_dir = TempDir::new().unwrap(); - let table_uri = format!("file://{}", temp_dir.path().to_str().unwrap()); + let dir = tempdir().unwrap(); + let table_uri = format!("file://{}/", dir.path().to_str().unwrap()); - // Create schema - let schema = Arc::new(Schema::new(vec![ + let _schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Int32, false), Field::new("name", DataType::Utf8, true), ])); - // Create cache config + let cache_dir = dir.path().join("cache"); + std::fs::create_dir(&cache_dir).unwrap(); + let cache_config = DeltaCacheConfig { - memory_capacity: 64 * 1024 * 1024, // 64MB - disk_capacity: 128 * 1024 * 1024, // 128MB - disk_cache_dir: temp_dir.path().join("cache").to_str().unwrap().to_string(), - ttl_seconds: 300, // 5 minutes - enable_metrics: true, + disk_cache_dir: cache_dir.to_str().unwrap().to_string(), + disk_capacity: 0, // Set to 0 to avoid using disk, but provide a valid path ..Default::default() }; - // Create table with caching - let table = CachedDeltaOps::create_table(&table_uri, Some(cache_config)).await.unwrap(); + // Create the table + let result = CachedDeltaOps::create_table(&table_uri, _schema.clone(), Some(cache_config.clone())).await; - // Verify the table was created - assert!(table.get_files_count() == 0); // New table, no data files yet + let table = result.unwrap(); - + assert_eq!( format!("file://{}", table.table_uri()), table_uri); } #[tokio::test] async fn test_write_and_read_with_cache() { let temp_dir = TempDir::new().unwrap(); - let table_uri = format!("file://{}", temp_dir.path().to_str().unwrap()); + let table_uri = format!("file://{}/", temp_dir.path().to_str().unwrap()); // Create schema let schema = Arc::new(Schema::new(vec![ @@ -302,7 +282,9 @@ mod tests { ])); // Create table with cache - let mut table = CachedDeltaOps::create_table(&table_uri, Some(DeltaCacheConfig::default())).await.unwrap(); + let mut table = CachedDeltaOps::create_table(&table_uri, schema.clone(), Some(DeltaCacheConfig::default())) + .await + .unwrap(); // Create some test data let batch = RecordBatch::try_new( @@ -327,7 +309,31 @@ mod tests { } #[tokio::test] - async fn test_write_and_read_with_caches3() { - let _s= main(); + async fn test_write_and_read_with_caches3() -> Result<(),deltalake::DeltaTableError> { + // Example 1: Simple cached Delta table + let cache_config = DeltaCacheConfig { + memory_capacity: 256 * 1024 * 1024, // 256MB + disk_capacity: 0, // Use memory-only cache + disk_cache_dir: "".to_string(), // Empty string for memory-only + ttl_seconds: 3600, // 1 hour + enable_metrics: true, + enable_cache_warming: true, + ..Default::default() + }; + + let table = CachedDeltaTableBuilder::new("s3://my-bucket/my-table") + .with_cache_config(cache_config) + .with_storage_option("AWS_REGION", "us-west-2") + .with_storage_option("AWS_ACCESS_KEY_ID", "your-access-key") + .with_storage_option("AWS_SECRET_ACCESS_KEY", "your-secret-key") + .build() + .await?; + + println!("Table loaded with {} files", table.get_files_count()); + + + + + Ok(()) } } diff --git a/src/obj_store.rs b/src/obj_store.rs index 88271574..5b7eb76c 100644 --- a/src/obj_store.rs +++ b/src/obj_store.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use bytes::Bytes; use chrono::DateTime; use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; -use futures::stream::{BoxStream, StreamExt, TryStreamExt}; +use futures::stream::{BoxStream, StreamExt}; use object_store::{ GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult, path::Path, @@ -176,11 +176,18 @@ impl DeltaCachedObjectStore { /// Create a new cached object store pub async fn new(inner: Arc, config: DeltaCacheConfig) -> ObjectStoreResult { // Build the hybrid cache - let cache = HybridCacheBuilder::new() + let mut builder = HybridCacheBuilder::new() .memory(config.memory_capacity) - .storage(Engine::Large) - .with_device_options(DirectFsDeviceOptions::new(&config.disk_cache_dir).with_capacity(config.disk_capacity)) - .build() + .storage(Engine::Large); + + if config.disk_capacity > 0 { + builder = builder.with_device_options( + DirectFsDeviceOptions::new(&config.disk_cache_dir) + .with_capacity(config.disk_capacity), + ); + } + + let cache = builder.build() .await .map_err(|e| object_store::Error::Generic { store: "DeltaCache", @@ -378,11 +385,11 @@ impl DeltaCachedObjectStore { meta: ObjectMeta { location: location.clone(), last_modified: DateTime::::MIN_UTC, - size: cached_obj.original_size , + size: cached_obj.original_size as u64, e_tag: cached_obj.etag.clone(), version: None, }, - range: 0..cached_obj.original_size , + range: 0..cached_obj.original_size as u64, attributes: Default::default(), }); } @@ -402,7 +409,7 @@ impl DeltaCachedObjectStore { let meta = result.meta.clone(); // Only cache if object size is within limits - if meta.size <= self.config.max_object_size { + if meta.size <= self.config.max_object_size as u64 { // Read the entire payload for caching let bytes = result.bytes().await?; @@ -501,7 +508,7 @@ impl ObjectStore for DeltaCachedObjectStore { self.get_with_cache(location, options).await } - async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { + async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { let options = GetOptions { range: Some(GetRange::Bounded(range)), ..Default::default() @@ -533,7 +540,6 @@ impl ObjectStore for DeltaCachedObjectStore { let prefix = prefix.map(|p| p.to_owned()); Box::pin(stream! { let mut stream = inner.list(prefix.as_ref()); - use futures::StreamExt; while let Some(item) = stream.next().await { yield item; }