diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..aa99c8ef --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +AWS_REGION= +AWS_S3_BUCKET= +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +PGWIRE_PORT=5432 +PORT=80 +TIMEFUSION_TABLE_PREFIX=timefusion + +# Batch insert configuration +# Interval between batch inserts in milliseconds (default: 1000) +BATCH_INTERVAL_MS=1000 +# Maximum number of rows to process in a single batch (default: 1000) +MAX_BATCH_SIZE=1000 +# Set to "true" to enable batching queue (default: false = direct insertion) +ENABLE_BATCH_QUEUE=false +# Maximum number of concurrent PostgreSQL connections (default: 100) +MAX_PG_CONNECTIONS=100 diff --git a/Cargo.lock b/Cargo.lock index 60d20a39..1414e9f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -366,9 +366,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "array-init" @@ -376,6 +382,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" +[[package]] +name = "array-util" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e509844de8f09b90a2c3444684a2b6695f4071360e13d2fda0af9f749cc2ed6" +dependencies = [ + "arrayvec", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -442,9 +457,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "54.3.0" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6ed265c73f134a583d02c3cab5e16afab9446d8048ede8707e31f85fad58a0" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" dependencies = [ "bytes", "half", @@ -490,9 +505,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "54.3.0" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f2cebf504bb6a92a134a87fff98f01b14fbb3a93ecf7aef90cd0f888c5fffa4" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" dependencies = [ "arrow-buffer", "arrow-schema", @@ -527,7 +542,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap", + "indexmap 2.9.0", "lexical-core", "num", "serde", @@ -562,9 +577,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "54.3.0" +version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c53775bba63f319189f366d2b86e9a8889373eb198f07d8544938fc9f8ed9a" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" dependencies = [ "bitflags 2.9.0", "serde", @@ -601,6 +616,18 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.4.19" @@ -618,6 +645,34 @@ dependencies = [ "zstd-safe", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.88" @@ -644,6 +699,18 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "auto_enums" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c170965892137a3a9aeb000b4524aa3cc022a310e709d848b6e1cdce4ab4781" +dependencies = [ + "derive_utils", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -694,9 +761,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.12.6" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -705,9 +772,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f" +checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" dependencies = [ "bindgen", "cc", @@ -744,9 +811,9 @@ dependencies = [ [[package]] name = "aws-sdk-dynamodb" -version = "1.70.0" +version = "1.71.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac281113af7f8700394bf25eb272b842b7ca088810e96c928f812282f2e6f44" +checksum = "2d49d08b1c99ca9a7de728a8975504857f2c24581a177f952e2a10244c305a1c" dependencies = [ "aws-credential-types", "aws-runtime", @@ -802,9 +869,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.63.0" +version = "1.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1cb45b83b53b5cd55ee33fd9fd8a70750255a3f286e4dca20e882052f2b256f" +checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" dependencies = [ "aws-credential-types", "aws-runtime", @@ -825,9 +892,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.64.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4d9bc075ea6238778ed3951b65d3cde8c3864282d64fdcd19f2a90c0609f1" +checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" dependencies = [ "aws-credential-types", "aws-runtime", @@ -848,9 +915,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.64.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819ccba087f403890fee4825eeab460e64c59345667d2b83a12cf544b581e3a7" +checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" dependencies = [ "aws-credential-types", "aws-runtime", @@ -974,7 +1041,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.4.8", + "h2 0.4.9", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", @@ -985,7 +1052,7 @@ dependencies = [ "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -1115,9 +1182,9 @@ dependencies = [ [[package]] name = "backon" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "970d91570c01a8a5959b36ad7dd1c30642df24b6b3068710066f6809f7033bb7" +checksum = "fd0b50b1b78dbadd44ab18b3c794e496f3a139abb9fbc27d9c94c4eebbb96496" dependencies = [ "fastrand", "tokio", @@ -1187,9 +1254,9 @@ dependencies = [ [[package]] name = "bigdecimal" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c" +checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" dependencies = [ "autocfg", "libm", @@ -1274,9 +1341,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17679a8d69b6d7fd9cd9801a536cec9fa5e5970b69f9d4747f70b39b031f5e7" +checksum = "389a099b34312839e16420d499a9cad9650541715937ffbdd40d36f49e77eeb3" dependencies = [ "arrayref", "arrayvec", @@ -1454,9 +1521,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.17" +version = "1.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" dependencies = [ "jobserver", "libc", @@ -1570,9 +1637,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.34" +version = "4.5.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e958897981290da2a852763fe9cdb89cd36977a5d729023127095fa94d95e2ff" +checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04" dependencies = [ "clap_builder", "clap_derive", @@ -1580,14 +1647,14 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.34" +version = "4.5.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b0f35019843db2160b5bb19ae09b4e6411ac33fc6a712003c33e03090e2489" +checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] @@ -1617,6 +1684,15 @@ dependencies = [ "cc", ] +[[package]] +name = "cmsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c840ee51da812c6cd621f9f7e07dfb00a49f91283a8e6380c78cba4f61aba" +dependencies = [ + "paste", +] + [[package]] name = "color-eyre" version = "0.6.3" @@ -1660,6 +1736,15 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1823,6 +1908,17 @@ dependencies = [ "itertools 0.10.5", ] +[[package]] +name = "cron" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8c3e73077b4b4a6ab1ea5047c37c57aee77657bc8ecd6f29b0af082d0b0c07" +dependencies = [ + "chrono", + "nom", + "once_cell", +] + [[package]] name = "crontab" version = "0.1.0" @@ -1833,6 +1929,28 @@ dependencies = [ "time 0.1.45", ] +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1852,6 +1970,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1917,6 +2044,76 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.100", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.100", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -2036,7 +2233,7 @@ dependencies = [ "base64 0.22.1", "half", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.9.0", "libc", "log 0.4.27", "object_store", @@ -2131,7 +2328,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap", + "indexmap 2.9.0", "paste", "recursive", "serde_json", @@ -2146,7 +2343,7 @@ checksum = "18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e" dependencies = [ "arrow", "datafusion-common", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "paste", ] @@ -2312,7 +2509,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "log 0.4.27", "recursive", @@ -2335,7 +2532,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "log 0.4.27", "paste", @@ -2397,7 +2594,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "log 0.4.27", "parking_lot 0.12.3", @@ -2408,7 +2605,7 @@ dependencies = [ [[package]] name = "datafusion-postgres" version = "0.3.0" -source = "git+https://github.com/apitoolkit/datafusion-postgres.git?branch=insert-query-compliance#a5ad9c9fd523e6a52b3fabd0889e0d612b8e3cd3" +source = "git+https://github.com/sunng87/datafusion-postgres.git?rev=2cf58787a8bf3e12a82b836d7dbdc5f6aee9f5a6#2cf58787a8bf3e12a82b836d7dbdc5f6aee9f5a6" dependencies = [ "async-trait", "chrono", @@ -2454,7 +2651,7 @@ dependencies = [ "bigdecimal", "datafusion-common", "datafusion-expr", - "indexmap", + "indexmap 2.9.0", "log 0.4.27", "recursive", "regex 1.11.1", @@ -2463,8 +2660,8 @@ dependencies = [ [[package]] name = "datafusion-uwheel" -version = "40.0.0" -source = "git+https://github.com/apitoolkit/datafusion-uwheel.git?branch=datafusion-46#7d7d1223470f06d1ae8732929f40d61e8698ec57" +version = "46.0.0" +source = "git+https://github.com/apitoolkit/datafusion-uwheel.git?branch=datafusion-46#053dada166281ab3030ec9edcaaf39af741d09bb" dependencies = [ "bitpacking", "chrono", @@ -2485,7 +2682,7 @@ dependencies = [ "fix-hidden-lifetime-bug", "futures", "home", - "indexmap", + "indexmap 2.9.0", "itertools 0.13.0", "object_store", "parquet", @@ -2522,7 +2719,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78889f4005974b848f130fa5dedae81987f1bc93b107291ea87d900c93b6c3bb" dependencies = [ "deltalake-aws", + "deltalake-azure", "deltalake-core", + "deltalake-gcp", ] [[package]] @@ -2552,6 +2751,24 @@ dependencies = [ "uuid", ] +[[package]] +name = "deltalake-azure" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfbea4786321ebd88e083ec74ce513ec7fcad9ddc880b611770dee012652567" +dependencies = [ + "async-trait", + "bytes", + "deltalake-core", + "futures", + "object_store", + "regex 1.11.1", + "thiserror 2.0.12", + "tokio", + "tracing", + "url", +] + [[package]] name = "deltalake-core" version = "0.25.0" @@ -2589,7 +2806,7 @@ dependencies = [ "fix-hidden-lifetime-bug", "futures", "humantime", - "indexmap", + "indexmap 2.9.0", "itertools 0.14.0", "libc", "maplit", @@ -2617,6 +2834,24 @@ dependencies = [ "z85", ] +[[package]] +name = "deltalake-gcp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa80de5b3e9e53eb9a98d976ac1d64a70b798a73d63cdd83497cc317a5063602" +dependencies = [ + "async-trait", + "bytes", + "deltalake-core", + "futures", + "object_store", + "regex 1.11.1", + "thiserror 2.0.12", + "tokio", + "tracing", + "url", +] + [[package]] name = "der" version = "0.6.1" @@ -2629,11 +2864,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -2681,6 +2917,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "derive_utils" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "digest" version = "0.10.7" @@ -2709,6 +2956,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dunce" version = "1.0.5" @@ -2806,9 +3059,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" dependencies = [ "anstream", "anstyle", @@ -2825,9 +3078,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", "windows-sys 0.59.0", @@ -2839,6 +3092,27 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "eyre" version = "0.6.12" @@ -2909,12 +3183,24 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "miniz_oxide 0.8.5", + "miniz_oxide 0.8.8", +] + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "spin", ] [[package]] @@ -2953,6 +3239,116 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "635c7077026867cb5e5ea576c461f29b1c4151fce7a9d7cc3a1b1a9902d95c65" +dependencies = [ + "anyhow", + "equivalent", + "foyer-common", + "foyer-memory", + "foyer-storage", + "madsim-tokio", + "mixtrics", + "pin-project", + "serde", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-common" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ed2316785e80137c7b91bb74dab1dc1967c3272df05825397b73ae8fc527041" +dependencies = [ + "ahash 0.8.11", + "bincode", + "bytes", + "cfg-if", + "itertools 0.14.0", + "madsim-tokio", + "mixtrics", + "parking_lot 0.12.3", + "pin-project", + "serde", + "thiserror 2.0.12", + "tokio", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090cf5b89d49fd61e7da9bfae3a1aef605f03196d542b2f8171c74f3add013f4" +dependencies = [ + "ahash 0.8.11", + "arc-swap", + "bitflags 2.9.0", + "cmsketch", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "hashbrown 0.15.2", + "itertools 0.14.0", + "madsim-tokio", + "mixtrics", + "parking_lot 0.12.3", + "pin-project", + "serde", + "thiserror 2.0.12", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-storage" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "095e857c97d6339d4a4a6424b88d08fe08ad0366bfbfaf65d6ddf55baf3d2a38" +dependencies = [ + "ahash 0.8.11", + "allocator-api2", + "anyhow", + "array-util", + "auto_enums", + "bytes", + "clap", + "equivalent", + "flume", + "foyer-common", + "foyer-memory", + "fs4", + "futures-core", + "futures-util", + "itertools 0.14.0", + "libc", + "lz4", + "madsim-tokio", + "ordered_hash_map", + "parking_lot 0.12.3", + "paste", + "pin-project", + "rand 0.9.0", + "serde", + "thiserror 2.0.12", + "tokio", + "tracing", + "twox-hash 2.1.0", + "zstd", +] + [[package]] name = "fs-err" version = "3.1.0" @@ -2972,6 +3368,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix 1.0.5", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -3154,7 +3560,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", @@ -3163,9 +3569,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" dependencies = [ "atomic-waker", "bytes", @@ -3173,7 +3579,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", @@ -3182,9 +3588,9 @@ dependencies = [ [[package]] name = "half" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "bytemuck", "cfg-if", @@ -3201,6 +3607,15 @@ dependencies = [ "ahash 0.7.8", ] +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash 0.8.11", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -3377,7 +3792,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.8", + "h2 0.4.9", "http 1.3.1", "http-body 1.0.1", "httparse", @@ -3414,7 +3829,7 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -3440,9 +3855,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-channel", @@ -3450,6 +3865,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "hyper 1.6.0", + "libc", "pin-project-lite", "socket2", "tokio", @@ -3459,9 +3875,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.62" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3599,6 +4015,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -3634,12 +4056,24 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "indexmap" -version = "2.8.0" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown 0.15.2", + "serde", ] [[package]] @@ -3739,9 +4173,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260" +checksum = "e5ad87c89110f55e4cd4dc2893a9790820206729eaf221555f742d540b0724a0" dependencies = [ "jiff-static", "log 0.4.27", @@ -3752,9 +4186,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.5" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" +checksum = "d076d5b64a7e2fe6f0743f02c43ca4a6725c0f904203bfe276a5b3e793103605" dependencies = [ "proc-macro2", "quote", @@ -3778,10 +4212,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.2", "libc", ] @@ -3902,9 +4337,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.171" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libloading" @@ -3942,9 +4377,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" @@ -4003,13 +4438,32 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lz4_flex" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" dependencies = [ - "twox-hash", + "twox-hash 1.6.3", ] [[package]] @@ -4023,6 +4477,60 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "madsim" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3c97f34bb19cf6a435a4da2187e90acc6bc59faa730e493b28b6d33e1bb9ccb" +dependencies = [ + "ahash 0.8.11", + "async-channel", + "async-stream", + "async-task", + "bincode", + "bytes", + "downcast-rs", + "futures-util", + "lazy_static", + "libc", + "madsim-macros", + "naive-timer", + "panic-message", + "rand 0.8.5", + "rand_xoshiro", + "rustversion", + "serde", + "spin", + "tokio", + "tokio-util", + "toml", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "madsim-macros" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "madsim-tokio" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3eb2acc57c82d21d699119b859e2df70a91dbdb84734885a1e72be83bdecb5" +dependencies = [ + "madsim", + "spin", + "tokio", +] + [[package]] name = "maplit" version = "1.0.2" @@ -4031,9 +4539,9 @@ checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" [[package]] name = "marrow" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd5fc5916496c19f17c6b7b6ebc8210546f3fe42d59179efe78ac9562e09a2d6" +checksum = "3641f6a55539a8b6e5349b3bdfb5b315714fbceda3253815838f49e40e3ea757" dependencies = [ "arrow-array", "arrow-buffer", @@ -4084,6 +4592,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5" +dependencies = [ + "ahash 0.8.11", + "portable-atomic", +] + [[package]] name = "mime" version = "0.3.17" @@ -4117,9 +4635,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -4136,6 +4654,31 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mixtrics" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "749ed12bab176c8a42c13a679dd2de12876d5ad4abe7525548e31ae001a9ebbf" +dependencies = [ + "itertools 0.14.0", + "parking_lot 0.12.3", +] + +[[package]] +name = "naive-timer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" + +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.15", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -4212,6 +4755,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -4283,6 +4837,7 @@ dependencies = [ "bytes", "chrono", "futures", + "httparse", "humantime", "hyper 1.6.0", "itertools 0.13.0", @@ -4293,6 +4848,7 @@ dependencies = [ "rand 0.8.5", "reqwest", "ring", + "rustls-pemfile 2.2.0", "serde", "serde_json", "snafu", @@ -4304,9 +4860,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.2" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2806eaa3524762875e21c3dcd057bc4b7bfa01ce4da8d46be1cd43649e1cc6b" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "oorandom" @@ -4316,9 +4872,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.71" +version = "0.10.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" dependencies = [ "bitflags 2.9.0", "cfg-if", @@ -4348,9 +4904,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.106" +version = "0.9.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" dependencies = [ "cc", "libc", @@ -4445,6 +5001,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered_hash_map" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0e5f22bf6dd04abd854a8874247813a8fa2c8c1260eba6fbb150270ce7c176" +dependencies = [ + "hashbrown 0.13.2", +] + [[package]] name = "outref" version = "0.5.2" @@ -4480,6 +5045,18 @@ dependencies = [ "sha2", ] +[[package]] +name = "panic-message" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.11.2" @@ -4523,7 +5100,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.10", + "redox_syscall 0.5.11", "smallvec", "windows-targets 0.52.6", ] @@ -4560,7 +5137,7 @@ dependencies = [ "snap", "thrift", "tokio", - "twox-hash", + "twox-hash 1.6.3", "zstd", "zstd-sys", ] @@ -4593,7 +5170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.9.0", ] [[package]] @@ -4797,9 +5374,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", "syn 2.0.100", @@ -4877,9 +5454,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f1c6c3591120564d64db2261bec5f910ae454f01def849b9c22835a84695e86" +checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" dependencies = [ "cfg-if", "indoc", @@ -4896,9 +5473,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9b6c2b34cf71427ea37c7001aefbaeb85886a074795e35f161f5aecc7620a7a" +checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" dependencies = [ "once_cell", "target-lexicon", @@ -4906,9 +5483,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5507651906a46432cdda02cd02dd0319f6064f1374c9147c45b978621d2c3a9c" +checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" dependencies = [ "libc", "pyo3-build-config", @@ -4916,9 +5493,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d394b5b4fd8d97d48336bb0dd2aebabad39f1d294edd6bcd2cccf2eefe6f42" +checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -4928,9 +5505,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd72da09cfa943b1080f621f024d2ef7e2773df7badd51aa30a2be1f8caa7c8e" +checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" dependencies = [ "heck", "proc-macro2", @@ -4941,9 +5518,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.37.3" +version = "0.37.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf763ab1c7a3aa408be466efc86efe35ed1bd3dd74173ed39d6b0d0a6f0ba148" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" dependencies = [ "memchr", "serde", @@ -4961,7 +5538,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.25", + "rustls 0.23.26", "socket2", "thiserror 2.0.12", "tokio", @@ -4980,7 +5557,7 @@ dependencies = [ "rand 0.9.0", "ring", "rustc-hash 2.1.1", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -5084,6 +5661,15 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rayon" version = "1.10.0" @@ -5135,9 +5721,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ "bitflags 2.9.0", ] @@ -5235,7 +5821,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.8", + "h2 0.4.9", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -5252,7 +5838,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -5331,9 +5917,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.10" +version = "0.10.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652edd001c53df0b3f96a36a8dc93fce6866988efc16808235653c6bcac8bf2" +checksum = "19e8d2cfa184d94d0726d650a9f4a1be7f9b76ac9fdb954219878dc00c1c1e7b" dependencies = [ "bytemuck", "byteorder", @@ -5398,14 +5984,14 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys 0.9.3", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] @@ -5423,9 +6009,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.25" +version = "0.23.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" +checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" dependencies = [ "aws-lc-rs", "log 0.4.27", @@ -5650,9 +6236,9 @@ dependencies = [ [[package]] name = "serde_arrow" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c51448819179b0656880c6b83a7d53c427039cdfcc300dfa0c8a7974c87ce39" +checksum = "a0462b8e06478cd310e8de11ea2e64c214522275a0b537b3879dbed24a9e01b5" dependencies = [ "arrow-array", "arrow-schema", @@ -5686,6 +6272,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5698,6 +6293,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.9.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time 0.3.41", +] + +[[package]] +name = "serde_with_macros" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "serial_test" version = "3.2.0" @@ -5824,9 +6449,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "snafu" @@ -5857,14 +6482,23 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.6.0" @@ -5976,6 +6610,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -6116,7 +6756,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", - "rustix 1.0.3", + "rustix 1.0.5", "windows-sys 0.59.0", ] @@ -6251,6 +6891,7 @@ dependencies = [ "anyhow", "arrow", "arrow-schema", + "async-stream", "async-trait", "aws-config", "aws-sdk-s3", @@ -6261,6 +6902,7 @@ dependencies = [ "chrono", "color-eyre", "criterion", + "crossbeam", "datafusion", "datafusion-common", "datafusion-functions-json", @@ -6270,30 +6912,38 @@ dependencies = [ "deltalake", "dotenv", "env_logger", + "flate2", + "foyer", "futures", "lazy_static", "log 0.4.27", + "metrics", + "object_store", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", "pgwire", "rand 0.8.5", "regex 1.11.1", - "rustls 0.23.25", + "rustls 0.23.26", "rustls-pemfile 2.2.0", "scopeguard", "serde", "serde_arrow", "serde_json", + "serde_with", "serial_test", "sled", "sqllogictest", "sqlparser 0.55.0", + "tap", "task", "tempfile", "tokio", + "tokio-cron-scheduler", "tokio-postgres", "tokio-rustls 0.26.2", + "tokio-stream", "tokio-util", "tracing", "tracing-opentelemetry", @@ -6348,9 +6998,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.1" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", @@ -6364,6 +7014,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-cron-scheduler" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c2e3a88f827f597799cf70a6f673074e62f3fc5ba5993b2873345c618a29af" +dependencies = [ + "chrono", + "cron", + "num-derive", + "num-traits", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "tokio-macros" version = "2.5.0" @@ -6427,7 +7092,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.25", + "rustls 0.23.26", "tokio", ] @@ -6455,11 +7120,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -6467,7 +7147,9 @@ version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ - "indexmap", + "indexmap 2.9.0", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] @@ -6626,6 +7308,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "twox-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "typenum" version = "1.18.0" @@ -6987,7 +7678,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ - "redox_syscall 0.5.10", + "redox_syscall 0.5.11", "wasite", "web-sys", ] @@ -7025,11 +7716,37 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.52.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" dependencies = [ - "windows-targets 0.52.6", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings 0.4.0", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", ] [[package]] @@ -7045,7 +7762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", - "windows-strings", + "windows-strings 0.3.1", "windows-targets 0.53.0", ] @@ -7067,6 +7784,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -7215,9 +7941,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index cda8431f..c1a5340a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,32 +5,35 @@ edition = "2024" [dependencies] tokio = { version = "1.43", features = ["full"] } +foyer = {version ="0.17.3",features=["serde"]} +object_store = { version = "0.12.2", features = ["aws", "azure", "gcp"] } datafusion = "46.0.0" arrow = "54.2.0" uuid = { version = "1.13", features = ["v4", "serde"] } serde = { version = "1", features = ["derive"] } serde_arrow = { version = "0.13.1", features = ["arrow-54"] } serde_json = "1.0.138" +serde_with = "3.12" async-trait = "0.1.86" env_logger = "0.11.6" log = "0.4.25" color-eyre = "0.6.3" arrow-schema = "54.1.0" regex = "1.11.1" -deltalake = { version = "0.25.0", features = ["datafusion", "s3"] } +deltalake = { version = "0.26.2", features = ["datafusion", "s3","azure", "gcs",] } delta_kernel = { version = "0.8.0", features = [ "arrow-conversion", "default-engine", ] } -chrono = "0.4.39" +chrono = { version = "0.4.39", features = ["serde"] } pgwire = "0.28.0" futures = "0.3.31" bytes = "1.4" tokio-rustls = "0.26.1" sled = "0.34.7" actix-web = "4.9.0" -# datafusion-postgres = { git = "https://github.com/apitoolkit/FusionGate.git" } -datafusion-postgres = { git = "https://github.com/apitoolkit/datafusion-postgres.git", branch = "insert-query-compliance" } +datafusion-postgres = { git = "https://github.com/sunng87/datafusion-postgres.git", rev = "2cf58787a8bf3e12a82b836d7dbdc5f6aee9f5a6" } +# datafusion-postgres = { git = "https://github.com/apitoolkit/datafusion-postgres.git", branch = "insert-query-compliance" } # datafusion-postgres = { path = "../datafusion-projects/datafusion-postgres/datafusion-postgres/" } datafusion-functions-json = "0.46.0" anyhow = "1.0.95" @@ -39,9 +42,12 @@ tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } tracing = "0.1.41" dotenv = "0.15.0" task = "0.0.1" +crossbeam = "0.8.4" sqlparser = "0.55.0" rustls-pemfile = "2.2.0" rustls = "0.23.23" +tokio-stream = { version = "0.1.17", features = ["net"] } +tap = "1.0.1" actix-service = "2.0.2" lazy_static = "1.5.0" bcrypt = "0.17.0" @@ -62,6 +68,10 @@ aws-types = "1.3.6" aws-sdk-s3 = "1.3.0" url = "2.5.4" datafusion-common = "46.0.0" +tokio-cron-scheduler = "0.10" +metrics = "0.24.2" +flate2 = "1.1.1" +async-stream = "0.3" [dev-dependencies] serial_test = "3.2.0" @@ -70,5 +80,9 @@ scopeguard = "1.2.0" rand = "0.8.5" [features] -default = [] +default = ["s3", "azure", "gcs"] +s3 = ["deltalake/s3", "object_store/aws"] +azure = ["deltalake/azure", "object_store/azure"] +gcs = ["deltalake/gcs", "object_store/gcp"] test = [] + diff --git a/README.md b/README.md index f2eaa189..0243870f 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,53 @@ A very specialized timeseries database created for events, logs, traces and metr Its designed to allow users plug in their own s3 storage and buckets and have their stored to their accounts. This way, timefusion is used as a compute and cache engine, not primary data storage. +Timefusion speaks the postgres dialect, so you can insert and read from it using any postgres client or driver. + ## Configuration Timefusion can be configured using the following environment variables: -| Variable | Description | Default | -| --------------------- | ----------------------------- | -------------------------- | -| `PORT` | HTTP server port | `80` | -| `PGWIRE_PORT` | PostgreSQL wire protocol port | `5432` | -| `AWS_S3_BUCKET` | AWS S3 bucket name | Required | -| `AWS_S3_ENDPOINT` | AWS S3 endpoint URL | `https://s3.amazonaws.com` | -| AWS_ACCESS_KEY_ID | AWS access key | - | -| AWS_SECRET_ACCESS_KEY | AWS secret key | - | +| Variable | Description | Default | +| ---------------------- | ------------------------------------------------ | --------------------------- | +| `PORT` | HTTP server port | `80` | +| `PGWIRE_PORT` | PostgreSQL wire protocol port | `5432` | +| `AWS_S3_BUCKET` | AWS S3 bucket name | Required | +| `AWS_S3_ENDPOINT` | AWS S3 endpoint URL | `https://s3.amazonaws.com` | +| `AWS_ACCESS_KEY_ID` | AWS access key | - | +| `AWS_SECRET_ACCESS_KEY`| AWS secret key | - | +| `TIMEFUSION_TABLE_PREFIX` | Prefix for Delta tables | `timefusion` | +| `BATCH_INTERVAL_MS` | Interval between batch inserts in milliseconds | `1000` | +| `MAX_BATCH_SIZE` | Maximum number of rows in a single batch | `1000` | +| `ENABLE_BATCH_QUEUE` | Whether to use batch queue for inserts | `false` (direct insertion) | +| `MAX_PG_CONNECTIONS` | Maximum number of concurrent PostgreSQL connections | `100` | For local development, you can set `QUEUE_DB_PATH` to a location in your development environment. + +## Usage + +There currently exists only 1 table. otel_logs_and_spans. +You can access it via psql: eg if running locally: + +``` +$ psql "postgresql://postgres:postgres@localhost:12345/postgres" + +psql (16.8 (Homebrew), server 0.28.0) +WARNING: psql major version 16, server major version 0.28. + Some psql features might not work. +Type "help" for help. + +postgres=> insert into otel_logs_and_spans (name, id, project_id, hashes, timestamp, date) values ('name3', 'id2', 'pid3', ARRAY[], '2025-04-14 02:00:24.898000', '2025-04-14 02:00:24.898000'); +INSERT 0 1 + +postgres=> select name, id, project_id,timestamp from otel_logs_and_spans limit 10; + name | id | project_id | timestamp +-------------------------------------------------------------+--------------------------------------+--------------------------------------+---------------------------- + GET api/v1/validations/profundity-interior/(?P[^/.]+)/$ | 00000000-09ab-47bc-b628-2554626d1261 | 00000000-876e-41fa-be63-52d5bcfc037e | 2025-04-14 20:45:08.713740 + GET api/v1/validations/tire-pressure/(?P[^/.]+)/$ | 00000000-3d2a-445d-b7bf-3e56125b48d4 | 00000000-876e-41fa-be63-52d5bcfc037e | 2025-04-14 22:01:00.816390 + POST api/v1/validations/warnings-of-wear/$ | 00000000-4ced-48f4-830d-64d3531eb7f0 | 00000000-876e-41fa-be63-52d5bcfc037e | 2025-04-14 21:18:08.635637 + +``` + +``` + +``` diff --git a/benches/benchmarks.rs b/benches/benchmarks.rs index a03c6641..17441e23 100644 --- a/benches/benchmarks.rs +++ b/benches/benchmarks.rs @@ -34,51 +34,51 @@ fn bench_batch_ingestion(c: &mut Criterion) { let mut records = Vec::with_capacity(batch_size); for _ in 0..batch_size { records.push(IngestRecord { - table_name: "bench_table".to_string(), - project_id: "bench_project".to_string(), - id: Uuid::new_v4().to_string(), - version: 1, - event_type: "bench_event".to_string(), - timestamp: "2025-03-11T12:00:00Z".to_string(), - trace_id: "trace".to_string(), - span_id: "span".to_string(), - parent_span_id: None, - trace_state: None, - start_time: "2025-03-11T12:00:00Z".to_string(), - end_time: Some("2025-03-11T12:00:01Z".to_string()), - duration_ns: 1_000_000_000, - span_name: "span_name".to_string(), - span_kind: "client".to_string(), - span_type: "bench".to_string(), - status: None, - status_code: 0, - status_message: "OK".to_string(), - severity_text: None, - severity_number: 0, - host: "localhost".to_string(), - url_path: "/".to_string(), - raw_url: "/".to_string(), - method: "GET".to_string(), - referer: "".to_string(), - path_params: None, - query_params: None, - request_headers: None, - response_headers: None, - request_body: None, - response_body: None, - endpoint_hash: "hash".to_string(), - shape_hash: "shape".to_string(), - format_hashes: vec!["fmt".to_string()], - field_hashes: vec!["field".to_string()], - sdk_type: "rust".to_string(), - service_version: None, - attributes: None, - events: None, - links: None, - resource: None, + table_name: "bench_table".to_string(), + project_id: "bench_project".to_string(), + id: Uuid::new_v4().to_string(), + version: 1, + event_type: "bench_event".to_string(), + timestamp: "2025-03-11T12:00:00Z".to_string(), + trace_id: "trace".to_string(), + span_id: "span".to_string(), + parent_span_id: None, + trace_state: None, + start_time: "2025-03-11T12:00:00Z".to_string(), + end_time: Some("2025-03-11T12:00:01Z".to_string()), + duration_ns: 1_000_000_000, + span_name: "span_name".to_string(), + span_kind: "client".to_string(), + span_type: "bench".to_string(), + status: None, + status_code: 0, + status_message: "OK".to_string(), + severity_text: None, + severity_number: 0, + host: "localhost".to_string(), + url_path: "/".to_string(), + raw_url: "/".to_string(), + method: "GET".to_string(), + referer: "".to_string(), + path_params: None, + query_params: None, + request_headers: None, + response_headers: None, + request_body: None, + response_body: None, + endpoint_hash: "hash".to_string(), + shape_hash: "shape".to_string(), + format_hashes: vec!["fmt".to_string()], + field_hashes: vec!["field".to_string()], + sdk_type: "rust".to_string(), + service_version: None, + attributes: None, + events: None, + links: None, + resource: None, instrumentation_scope: None, - errors: None, - tags: vec!["tag".to_string()], + errors: None, + tags: vec!["tag".to_string()], }); } @@ -110,51 +110,51 @@ fn bench_insertion_range(c: &mut Criterion) { let mut records = Vec::with_capacity(size); for _ in 0..size { records.push(IngestRecord { - table_name: "bench_table".to_string(), - project_id: "bench_project".to_string(), - id: Uuid::new_v4().to_string(), - version: 1, - event_type: "bench_event".to_string(), - timestamp: "2025-03-11T12:00:00Z".to_string(), - trace_id: "trace".to_string(), - span_id: "span".to_string(), - parent_span_id: None, - trace_state: None, - start_time: "2025-03-11T12:00:00Z".to_string(), - end_time: Some("2025-03-11T12:00:01Z".to_string()), - duration_ns: 1_000_000_000, - span_name: "span_name".to_string(), - span_kind: "client".to_string(), - span_type: "bench".to_string(), - status: None, - status_code: 0, - status_message: "OK".to_string(), - severity_text: None, - severity_number: 0, - host: "localhost".to_string(), - url_path: "/".to_string(), - raw_url: "/".to_string(), - method: "GET".to_string(), - referer: "".to_string(), - path_params: None, - query_params: None, - request_headers: None, - response_headers: None, - request_body: None, - response_body: None, - endpoint_hash: "hash".to_string(), - shape_hash: "shape".to_string(), - format_hashes: vec!["fmt".to_string()], - field_hashes: vec!["field".to_string()], - sdk_type: "rust".to_string(), - service_version: None, - attributes: None, - events: None, - links: None, - resource: None, + table_name: "bench_table".to_string(), + project_id: "bench_project".to_string(), + id: Uuid::new_v4().to_string(), + version: 1, + event_type: "bench_event".to_string(), + timestamp: "2025-03-11T12:00:00Z".to_string(), + trace_id: "trace".to_string(), + span_id: "span".to_string(), + parent_span_id: None, + trace_state: None, + start_time: "2025-03-11T12:00:00Z".to_string(), + end_time: Some("2025-03-11T12:00:01Z".to_string()), + duration_ns: 1_000_000_000, + span_name: "span_name".to_string(), + span_kind: "client".to_string(), + span_type: "bench".to_string(), + status: None, + status_code: 0, + status_message: "OK".to_string(), + severity_text: None, + severity_number: 0, + host: "localhost".to_string(), + url_path: "/".to_string(), + raw_url: "/".to_string(), + method: "GET".to_string(), + referer: "".to_string(), + path_params: None, + query_params: None, + request_headers: None, + response_headers: None, + request_body: None, + response_body: None, + endpoint_hash: "hash".to_string(), + shape_hash: "shape".to_string(), + format_hashes: vec!["fmt".to_string()], + field_hashes: vec!["field".to_string()], + sdk_type: "rust".to_string(), + service_version: None, + attributes: None, + events: None, + links: None, + resource: None, instrumentation_scope: None, - errors: None, - tags: vec!["tag".to_string()], + errors: None, + tags: vec!["tag".to_string()], }); } diff --git a/src/batch_queue.rs b/src/batch_queue.rs new file mode 100644 index 00000000..f3afcf6e --- /dev/null +++ b/src/batch_queue.rs @@ -0,0 +1,157 @@ +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use anyhow::Result; +use crossbeam::queue::SegQueue; +use delta_kernel::arrow::record_batch::RecordBatch; +use tokio::{sync::RwLock, time::interval}; +use tracing::{error, info}; + +/// BatchQueue collects RecordBatches and processes them at intervals +#[derive(Debug)] +pub struct BatchQueue { + queue: Arc>, + is_shutting_down: Arc>, +} + +impl BatchQueue { + pub fn new(db: Arc, interval_ms: u64, max_rows: usize) -> Self { + let queue = Arc::new(SegQueue::new()); + let is_shutting_down = Arc::new(RwLock::new(false)); + + let queue_clone = Arc::clone(&queue); + let shutdown_flag = Arc::clone(&is_shutting_down); + + tokio::spawn(async move { + let mut ticker = interval(Duration::from_millis(interval_ms)); + + loop { + ticker.tick().await; + + if *shutdown_flag.read().await { + process_batches(&db, &queue_clone, max_rows).await; + break; + } + + process_batches(&db, &queue_clone, max_rows).await; + } + }); + + Self { queue, is_shutting_down } + } + + /// Add a batch to the queue + pub fn queue(&self, batch: RecordBatch) -> Result<()> { + if let Ok(flag) = self.is_shutting_down.try_read() { + if *flag { + return Err(anyhow::anyhow!("BatchQueue is shutting down")); + } + } + + self.queue.push(batch); + Ok(()) + } + + /// Signal shutdown and wait for queue to drain + pub async fn shutdown(&self) { + let mut guard = self.is_shutting_down.write().await; + *guard = true; + } +} + +/// Process batches from the queue +async fn process_batches(db: &Arc, queue: &Arc>, max_rows: usize) { + if queue.is_empty() { + return; + } + + let mut batches = Vec::new(); + let mut total_rows = 0; + + // Take batches up to max_rows + while !queue.is_empty() && total_rows < max_rows { + if let Some(batch) = queue.pop() { + total_rows += batch.num_rows(); + batches.push(batch); + } else { + break; + } + } + + if batches.is_empty() { + return; + } + + // Measure and log the insertion performance + let start = Instant::now(); + + // Use skip_queue=true to force direct insertion and avoid infinite loop + match db.insert_records_batch("", batches.clone(), true).await { + Ok(_) => { + let elapsed = start.elapsed(); + info!( + batches_count = batches.len(), + rows_count = total_rows, + duration_ms = elapsed.as_millis(), + "Batch insert completed" + ); + } + Err(e) => { + error!("Failed to insert batches: {}", e); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use chrono::Utc; + use tokio::time::sleep; + + use super::*; + use crate::{database::Database, persistent_queue::OtelLogsAndSpans}; + + #[tokio::test] + async fn test_batch_queue() -> Result<()> { + dotenv::dotenv().ok(); + let test_prefix = format!("test-batch-{}", uuid::Uuid::new_v4()); + unsafe { + std::env::set_var("TIMEFUSION_TABLE_PREFIX", &test_prefix); + } + + // Initialize DB + let db = Arc::new(Database::new().await?); + + // Create batch queue with short interval for testing + let batch_queue = BatchQueue::new(Arc::clone(&db), 100, 10); + + // Create test records and convert to RecordBatch + let now = Utc::now(); + let records = (0..5) + .map(|i| OtelLogsAndSpans { + project_id: "default".to_string(), + timestamp: now, + id: format!("test-{}", i), + hashes: vec![], + date: now.date_naive(), + ..Default::default() + }) + .collect::>(); + + let fields = OtelLogsAndSpans::fields()?; + let batch = serde_arrow::to_record_batch(&fields, &records)?; + + // Queue and process the batch + batch_queue.queue(batch)?; + sleep(Duration::from_millis(200)).await; + + // Shutdown queue + batch_queue.shutdown().await; + sleep(Duration::from_millis(200)).await; + + Ok(()) + } +} diff --git a/src/database.rs b/src/database.rs index 140764a6..169c0c6f 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1,48 +1,59 @@ -use crate::persistent_queue::OtelLogsAndSpans; +use std::{any::Any, collections::HashMap, env, fmt, net::SocketAddr, sync::Arc, time::Duration}; + use anyhow::Result; use arrow_schema::SchemaRef; use async_trait::async_trait; -use datafusion::arrow::array::Array; -use datafusion::common::not_impl_err; -use datafusion::common::SchemaExt; -use datafusion::execution::context::SessionContext; -use datafusion::execution::TaskContext; -use datafusion::logical_expr::{Expr, Operator, TableProviderFilterPushDown}; -use datafusion::physical_expr::intervals::utils::check_support; -use datafusion::physical_plan::insert::{DataSink, DataSinkExec}; -use datafusion::physical_plan::DisplayAs; -use datafusion::scalar::ScalarValue; use datafusion::{ + arrow::array::Array, catalog::Session, + common::{SchemaExt, not_impl_err}, datasource::{TableProvider, TableType}, error::{DataFusionError, Result as DFResult}, - logical_expr::{dml::InsertOp, BinaryExpr}, - physical_plan::{DisplayFormatType, ExecutionPlan, SendableRecordBatchStream}, + execution::{TaskContext, context::SessionContext}, + logical_expr::{BinaryExpr, Expr, Operator, TableProviderFilterPushDown, dml::InsertOp}, + physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, SendableRecordBatchStream, + insert::{DataSink, DataSinkExec}, + }, + scalar::ScalarValue, }; +use datafusion_postgres::{DfSessionService, HandlerFactory}; use delta_kernel::arrow::record_batch::RecordBatch; -use deltalake::checkpoints; -use deltalake::{storage::StorageOptions, DeltaOps, DeltaTable, DeltaTableBuilder}; +use deltalake::{ + DeltaOps, DeltaTable, DeltaTableBuilder, checkpoints, + datafusion::parquet::{ + basic::{Compression, ZstdLevel}, + file::properties::WriterProperties, + }, + operations::transaction::CommitProperties, + storage::StorageOptions, +}; use futures::StreamExt; -use std::fmt; -use std::{any::Any, collections::HashMap, env, sync::Arc}; -use tokio::sync::RwLock; +use tokio::{net::TcpListener, sync::RwLock, time::timeout}; +use tokio_stream::wrappers::TcpListenerStream; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info}; use url::Url; +use crate::persistent_queue::OtelLogsAndSpans; + type ProjectConfig = (String, StorageOptions, Arc>); pub type ProjectConfigs = Arc>>; #[derive(Debug)] pub struct Database { - project_configs: ProjectConfigs, + project_configs: ProjectConfigs, + batch_queue: Option>, + maintenance_shutdown: Arc, } impl Clone for Database { fn clone(&self) -> Self { Self { - project_configs: Arc::clone(&self.project_configs), + project_configs: Arc::clone(&self.project_configs), + batch_queue: self.batch_queue.clone(), + maintenance_shutdown: Arc::clone(&self.maintenance_shutdown), } } } @@ -65,7 +76,9 @@ impl Database { let project_configs = HashMap::new(); let db = Self { - project_configs: Arc::new(RwLock::new(project_configs)), + project_configs: Arc::new(RwLock::new(project_configs)), + batch_queue: None, // Batch queue is set later + maintenance_shutdown: Arc::new(CancellationToken::new()), }; db.register_project("default", &storage_uri, None, None, None).await?; @@ -73,10 +86,72 @@ impl Database { Ok(db) } + /// Set the batch queue to use for insert operations + pub fn with_batch_queue(mut self, batch_queue: Arc) -> Self { + self.batch_queue = Some(batch_queue); + self + } + + /// Start background maintenance schedulers for optimize and vacuum operations + pub async fn start_maintenance_schedulers(self) -> Result { + use tokio_cron_scheduler::{Job, JobScheduler}; + + let scheduler = JobScheduler::new().await?; + let db = Arc::new(self.clone()); + + // Optimize job - every 3 hours + let optimize_job = Job::new_async("0 0 */3 * * *", { + let db = db.clone(); + move |_, _| { + let db = db.clone(); + Box::pin(async move { + info!("Running scheduled optimize on all tables"); + for (project_id, (_, _, table)) in db.project_configs.read().await.iter() { + if let Err(e) = db.optimize_table(table).await { + error!("Optimize failed for {}: {}", project_id, e); + } + } + }) + } + })?; + + scheduler.add(optimize_job).await?; + + // Vacuum job - daily at 3AM + let vacuum_job = Job::new_async("0 0 3 * * *", { + let db = db.clone(); + move |_, _| { + let db = db.clone(); + Box::pin(async move { + info!("Running scheduled vacuum on all tables"); + let retention_hours = env::var("TIMEFUSION_VACUUM_RETENTION_HOURS").unwrap_or_else(|_| "336".to_string()).parse::().unwrap_or(336); + + for (project_id, (_, _, table)) in db.project_configs.read().await.iter() { + info!("Vacuuming {} (retention: {}h)", project_id, retention_hours); + db.vacuum_table(table, retention_hours).await; + } + }) + } + })?; + + scheduler.add(vacuum_job).await?; + + // Start the scheduler + scheduler.start().await?; + + // Handle shutdown + let shutdown = self.maintenance_shutdown.clone(); + tokio::spawn(async move { + shutdown.cancelled().await; + info!("Shutting down maintenance scheduler"); + }); + + Ok(self) + } + /// Create and configure a SessionContext with DataFusion settings pub fn create_session_context(&self) -> SessionContext { - use datafusion::config::ConfigOptions; - use datafusion::execution::context::SessionContext; + use datafusion::{config::ConfigOptions, execution::context::SessionContext}; let mut options = ConfigOptions::new(); let _ = options.set("datafusion.sql_parser.enable_information_schema", "true"); @@ -89,7 +164,12 @@ impl Database { // Create tables and register them with session context let schema = OtelLogsAndSpans::schema_ref(); - let routing_table = ProjectRoutingTable::new("default".to_string(), Arc::new(self.clone()), schema); + + // Get batch queue from the app state if available + let batch_queue = self.batch_queue.as_ref().map(Arc::clone); + + let routing_table = ProjectRoutingTable::new("default".to_string(), Arc::new(self.clone()), schema, batch_queue); + ctx.register_table(OtelLogsAndSpans::table_name(), Arc::new(routing_table))?; info!("Registered ProjectRoutingTable with SessionContext"); @@ -101,9 +181,11 @@ impl Database { /// Register PostgreSQL settings table for compatibility pub fn register_pg_settings_table(&self, ctx: &SessionContext) -> datafusion::error::Result<()> { - use datafusion::arrow::array::StringArray; - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::arrow::record_batch::RecordBatch; + use datafusion::arrow::{ + array::StringArray, + datatypes::{DataType, Field, Schema}, + record_batch::RecordBatch, + }; let schema = Arc::new(Schema::new(vec![ Field::new("name", DataType::Utf8, false), @@ -146,9 +228,13 @@ impl Database { /// Register set_config UDF for PostgreSQL compatibility pub fn register_set_config_udf(&self, ctx: &SessionContext) { - use datafusion::arrow::array::{StringArray, StringBuilder}; - use datafusion::arrow::datatypes::DataType; - use datafusion::logical_expr::{create_udf, ColumnarValue, ScalarFunctionImplementation, Volatility}; + use datafusion::{ + arrow::{ + array::{StringArray, StringBuilder}, + datatypes::DataType, + }, + logical_expr::{ColumnarValue, ScalarFunctionImplementation, Volatility, create_udf}, + }; let set_config_fn: ScalarFunctionImplementation = Arc::new(move |args: &[ColumnarValue]| -> datafusion::error::Result { let param_value_array = match &args[1] { @@ -178,79 +264,78 @@ impl Database { ctx.register_udf(set_config_udf); } - /// Start a PGWire server with the given session context pub async fn start_pgwire_server( - &self, session_context: SessionContext, port: u16, shutdown_token: CancellationToken, - ) -> Result> { - use datafusion_postgres::{DfSessionService, HandlerFactory}; - use tokio::net::TcpListener; - - let pg_service = Arc::new(DfSessionService::new(session_context)); - let handler_factory = Arc::new(HandlerFactory(pg_service.clone())); - - info!("Attempting to bind PGWire server to 0.0.0.0:{}", port); - let bind_addr = format!("0.0.0.0:{}", port); - let pg_listener = match TcpListener::bind(&bind_addr).await { - Ok(listener) => { - info!("PGWire server successfully bound to {}", bind_addr); - listener - } - Err(e) => { - error!("Failed to bind PGWire server to {}: {:?}", bind_addr, e); - return Err(anyhow::anyhow!("Failed to bind PGWire server: {:?}", e)); - } - }; + &self, session_ctx: SessionContext, port: u16, shutdown: CancellationToken, + ) -> anyhow::Result> { + // 1) build listener + // Simple binding with clear logging + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + info!("Binding PGWire server to {}...", addr); + + // Use standard tokio TcpListener + let listener = TcpListener::bind(addr).await?; + + // Log successful binding + if let Ok(local_addr) = listener.local_addr() { + info!("PGWire server successfully bound to {}", local_addr); + } - // Log all local addresses this process is listening on - info!("PGWire server running on 0.0.0.0:{}", port); - info!("PGWire server local address: {:?}", pg_listener.local_addr()); + // 2) pgwire service + handler + let service = Arc::new(DfSessionService::new(session_ctx)); + let factory = Arc::new(HandlerFactory(service)); - let pgwire_shutdown = shutdown_token.clone(); + // 3) concurrency + logging + let max_conn = std::env::var("MAX_PG_CONNECTIONS").ok().and_then(|v| v.parse().ok()).unwrap_or(100) as usize; + info!("PGWire listening on 0.0.0.0:{} (limit {})", port, max_conn); - let pg_server = tokio::spawn({ - let handler_factory = handler_factory.clone(); + // 4) spawn the accept‐&‐process loop + let handle = tokio::spawn({ + let shutdown = shutdown.clone(); + let stream = TcpListenerStream::new(listener); async move { - loop { - tokio::select! { - _ = pgwire_shutdown.cancelled() => { - info!("PGWire server shutting down."); - break; - } - result = pg_listener.accept() => { - match result { - Ok((socket, addr)) => { - info!("PGWire: Received connection from {}, preparing to process", addr); - info!("PGWire: Socket details - local addr: {:?}, peer addr: {:?}", - socket.local_addr().map_err(|e| debug!("Failed to get local addr: {:?}", e)), - socket.peer_addr().map_err(|e| debug!("Failed to get peer addr: {:?}", e)) - ); - - let handler_factory = handler_factory.clone(); - tokio::spawn(async move { - info!("PGWire: Started processing connection from {}", addr); - match pgwire::tokio::process_socket(socket, None, handler_factory).await { - Ok(()) => { - info!("PGWire: Connection from {} processed successfully", addr); - } - Err(e) => { - error!("PGWire: Error processing connection from {}: {:?}", addr, e); - error!("PGWire: Error details - {}", e); - } - } - }); + stream + .take_until(shutdown.cancelled()) + .for_each_concurrent(max_conn, |conn| async { + match conn { + Ok(sock) => { + // Set TCP nodelay option for better performance + if let Err(e) = sock.set_nodelay(true) { + error!("Failed to set TCP_NODELAY: {}", e); + } + + // Log client connection info + if let Ok(peer_addr) = sock.peer_addr() { + info!("Client connected from {}", peer_addr); } - Err(e) => { - error!("PGWire: Error accepting connection: {:?}", e); - error!("PGWire: Connection accept error details - {}", e); + + // Use a longer timeout to prevent idle disconnections + let timeout_duration = Duration::from_secs(3600); // 1 hour + info!("Starting PGWire connection processing"); + let start_time = std::time::Instant::now(); + + match timeout(timeout_duration, pgwire::tokio::process_socket(sock, None, factory.clone())).await { + Ok(Ok(_)) => { + let elapsed = start_time.elapsed(); + info!("PGWire connection completed successfully (duration: {:?})", elapsed); + } + Ok(Err(e)) => { + let elapsed = start_time.elapsed(); + error!("PGWire connection error after {:?}: {}", elapsed, e); + } + Err(_) => { + error!("PGWire connection timed out after 1 hour"); + } } } + Err(e) => error!("TCP accept error: {}", e), } - } - } + }) + .await; + info!("PGWire server shut down."); } }); - Ok(pg_server) + Ok(handle) } pub async fn resolve_table(&self, project_id: &str) -> DFResult>> { @@ -299,48 +384,51 @@ impl Database { ))) } - pub async fn insert_records_batch(&self, _table: &str, batch: Vec) -> Result<()> { + pub async fn insert_records_batch(&self, _table: &str, batches: Vec, skip_queue: bool) -> Result<()> { + // Check if we should use the batch queue based on: + // 1. skip_queue parameter (if true, always skip) + // 2. ENABLE_BATCH_QUEUE env var (if set to "true", allow queue usage) + // 3. batch_queue existence + let enable_queue = env::var("ENABLE_BATCH_QUEUE").unwrap_or_else(|_| "false".to_string()) == "true"; + + if !skip_queue && enable_queue && self.batch_queue.is_some() { + let queue = self.batch_queue.as_ref().unwrap(); + // Add to batch queue + for batch in batches { + if let Err(e) = queue.queue(batch) { + return Err(anyhow::anyhow!("Queue error: {}", e)); + } + } + return Ok(()); + } + + // Direct insert logic if skip_queue=true, queue disabled, no batch queue, or when processing from batch queue let (_conn_str, _options, table_ref) = { let configs = self.project_configs.read().await; configs.get("default").ok_or_else(|| anyhow::anyhow!("Project ID '{}' not found", "default"))?.clone() }; + // Create writer properties with ZSTD compression level 6 and bloom filters + let writer_properties = WriterProperties::builder() + .set_compression(Compression::ZSTD(ZstdLevel::try_new(6).unwrap())) + .set_bloom_filter_enabled(true) + .set_sorting_columns(Some(OtelLogsAndSpans::sorting_columns())) + .build(); + // Scope the write lock to minimize lock time - let should_checkpoint = { + { let mut table = table_ref.write().await; // Create the DeltaOps with a clone of the table - let write_op = DeltaOps(table.clone()).write(batch).with_partition_columns(OtelLogsAndSpans::partitions()); + let write_op = DeltaOps(table.clone()) + .write(batches) + .with_partition_columns(OtelLogsAndSpans::partitions()) + .with_writer_properties(writer_properties); let new_table = write_op.await?; - let version = new_table.version(); *table = new_table; - version > 0 && version % 40 == 0 - }; - - // Checkpoint in the background if needed - if should_checkpoint { - // Clone the necessary resources for the background task - let table_ref_clone = Arc::clone(&table_ref); - - // Spawn a background task to perform checkpointing - tokio::spawn(async move { - // Take a read lock for checkpointing - let result = async { - let table = table_ref_clone.read().await; - let version = table.version(); - info!("Starting background checkpointing for Delta table at version {}", version); - checkpoints::create_checkpoint(&table, None).await - }.await; - - match result { - Ok(_) => info!("Background checkpointing completed successfully"), - Err(e) => error!("Background checkpointing failed: {}", e), - } - }); - - info!("Checkpoint scheduled in background"); + // Note: Checkpointing, optimization, and vacuum are now managed by scheduled jobs } Ok(()) @@ -353,14 +441,100 @@ impl Database { // Records should be grouped by span, and separated into groups then inserted into the // correct table. - use serde_arrow::schema::SchemaLike; - // Convert OtelLogsAndSpans records to Arrow RecordBatch format - let fields = Vec::::from_type::(serde_arrow::schema::TracingOptions::default())?; + let fields = OtelLogsAndSpans::fields()?; let batch = serde_arrow::to_record_batch(&fields, &records)?; // Call insert_records_batch with the converted batch to reuse common insertion logic - self.insert_records_batch("default", vec![batch]).await + // In tests we always skip the queue for direct insertion + self.insert_records_batch("default", vec![batch], true).await + } + + /// Optimize the Delta table using Z-ordering on timestamp and id columns + /// This improves query performance for time-based queries + async fn optimize_table(&self, table_ref: &Arc>) -> Result<()> { + // Log the start of the optimization operation + info!("Starting Delta table optimization with Z-ordering"); + + // Get a clone of the table to avoid holding the lock during the operation + let table_clone = { + let table = table_ref.read().await; + table.clone() + }; + + // Run optimize operation with Z-order on the timestamp and id columns + // and a target size of 256MB for optimal file size + let writer_properties = WriterProperties::builder() + .set_compression(Compression::ZSTD(ZstdLevel::try_new(6).unwrap())) + .set_bloom_filter_enabled(true) + .set_sorting_columns(Some(OtelLogsAndSpans::sorting_columns())) + .build(); + + // Note: Z-order functionality is achieved through sorting_columns in writer_properties + let optimize_result = DeltaOps(table_clone) + .optimize() + .with_type(deltalake::operations::optimize::OptimizeType::ZOrder(OtelLogsAndSpans::z_order_columns())) + .with_target_size(268435456) // 256MB + .with_writer_properties(writer_properties) + .await; + + match optimize_result { + Ok((new_table, metrics)) => { + info!( + "Optimization with sorted columns completed: {} files removed, {} files added, {} partitions optimized, {} total files considered, {} files skipped", + metrics.num_files_removed, + metrics.num_files_added, + metrics.partitions_optimized, + metrics.total_considered_files, + metrics.total_files_skipped + ); + + // Update the table reference with the optimized version + let mut table = table_ref.write().await; + *table = new_table; + + Ok(()) + } + Err(e) => { + error!("Optimization operation failed: {}", e); + Err(anyhow::anyhow!("Table optimization failed: {}", e)) + } + } + } + + /// Vacuum the Delta table to clean up old files that are no longer needed + /// This reduces storage costs and improves query performance + async fn vacuum_table(&self, table_ref: &Arc>, retention_hours: u64) { + // Log the start of the vacuum operation + info!("Starting vacuum operation with retention period of {} hours", retention_hours); + + // Get a clone of the table to avoid holding the lock during the operation + let table_clone = { + let table = table_ref.read().await; + table.clone() + }; + + // Directly run vacuum without dry run to delete old files + match DeltaOps(table_clone) + .vacuum() + .with_retention_period(chrono::Duration::hours(retention_hours as i64)) + .with_enforce_retention_duration(false) // Allow deletion of files newer than default retention + .await + { + Ok((_, metrics)) => { + let files_deleted = metrics.files_deleted.len(); + info!("Vacuum completed successfully, deleted {} files", files_deleted); + + // Update the table reference with the vacuumed version + let mut table = table_ref.write().await; + if let Ok(()) = table.update().await { + info!("Table updated after vacuum"); + } else { + error!("Failed to update table after vacuum"); + } + } + Err(e) => error!("Vacuum operation failed: {}", e), + } } pub async fn register_project( @@ -382,7 +556,7 @@ impl Database { storage_options.0.insert("AWS_ALLOW_HTTP".to_string(), "true".to_string()); - let mut table = match DeltaTableBuilder::from_uri(conn_str).with_storage_options(storage_options.0.clone()).with_allow_http(true).load().await { + let table = match DeltaTableBuilder::from_uri(conn_str).with_storage_options(storage_options.0.clone()).with_allow_http(true).load().await { Ok(table) => { // Check if table needs checkpointing - use same threshold as in insert_records_batch let version = table.version(); @@ -399,11 +573,18 @@ impl Database { // Create the table with project_id partitioning only for now // Timestamp partitioning is likely causing issues with nanosecond precision let delta_ops = DeltaOps::try_from_uri(&conn_str).await?; + let commit_properties = CommitProperties::default().with_create_checkpoint(true).with_cleanup_expired_logs(Some(true)); + + // Create table with ZSTD compression and auto-optimization + // Note: z-ordering will be applied via sorting_columns in the writer properties delta_ops .create() .with_columns(OtelLogsAndSpans::columns().unwrap_or_default()) .with_partition_columns(OtelLogsAndSpans::partitions()) .with_storage_options(storage_options.0.clone()) + .with_commit_properties(commit_properties) + .with_configuration_property(deltalake::TableProperty::AutoOptimizeOptimizeWrite, Some("true")) + .with_configuration_property(deltalake::TableProperty::AutoOptimizeAutoCompact, Some("true")) .await? } }; @@ -417,16 +598,18 @@ impl Database { #[derive(Debug, Clone)] pub struct ProjectRoutingTable { default_project: String, - database: Arc, - schema: SchemaRef, + database: Arc, + schema: SchemaRef, + batch_queue: Option>, } impl ProjectRoutingTable { - pub fn new(default_project: String, database: Arc, schema: SchemaRef) -> Self { + pub fn new(default_project: String, database: Arc, schema: SchemaRef, batch_queue: Option>) -> Self { Self { default_project, database, schema, + batch_queue, } } @@ -509,16 +692,26 @@ impl DataSink for ProjectRoutingTable { } async fn write_all(&self, mut data: SendableRecordBatchStream, _context: &Arc) -> DFResult { - let mut new_batches = vec![]; let mut row_count = 0; + let mut batches = Vec::new(); + + // Collect all batches from the stream while let Some(batch) = data.next().await.transpose()? { row_count += batch.num_rows(); - new_batches.push(batch); + batches.push(batch); } + + if batches.is_empty() { + return Ok(0); + } + + // Let the database handle the queue decision with skip_queue=false + // This means it will use the queue if it's available and not disabled via env var self.database - .insert_records_batch("", new_batches) + .insert_records_batch("", batches, false) .await - .map_err(|e| DataFusionError::Execution(format!("Failed to insert records: {}", e)))?; + .map_err(|e| DataFusionError::Execution(format!("Insert error: {}", e)))?; + Ok(row_count as u64) } @@ -545,7 +738,7 @@ impl TableProvider for ProjectRoutingTable { // Create a physical plan from the logical plan. // Check that the schema of the plan matches the schema of this table. match self.schema().logically_equivalent_names_and_types(&input.schema()) { - Ok(_) => info!("Schema validation passed"), + Ok(_) => debug!("insert_into; Schema validation passed"), Err(e) => { error!("Schema validation failed: {}", e); return Err(e); @@ -580,8 +773,7 @@ impl TableProvider for ProjectRoutingTable { #[cfg(test)] mod tests { use chrono::{TimeZone, Utc}; - use datafusion::assert_batches_eq; - use datafusion::prelude::SessionContext; + use datafusion::{assert_batches_eq, prelude::SessionContext}; use dotenv::dotenv; use serial_test::serial; use uuid::Uuid; @@ -604,7 +796,12 @@ mod tests { datafusion_functions_json::register_all(&mut session_context)?; let schema = OtelLogsAndSpans::schema_ref(); - let routing_table = ProjectRoutingTable::new("default".to_string(), Arc::new(db.clone()), schema); + let routing_table = ProjectRoutingTable::new( + "default".to_string(), + Arc::new(db.clone()), + schema, + None, // No batch queue in tests + ); session_context.register_table(OtelLogsAndSpans::table_name(), Arc::new(routing_table))?; Ok((db, session_context, test_prefix)) @@ -618,6 +815,7 @@ mod tests { vec![ OtelLogsAndSpans { project_id: "test_project".to_string(), + // date: timestamp1.date_naive(), timestamp: timestamp1, observed_timestamp: Some(timestamp1), id: "span1".to_string(), @@ -632,6 +830,7 @@ mod tests { }, OtelLogsAndSpans { project_id: "test_project".to_string(), + // date: timestamp2.date_naive(), timestamp: timestamp2, observed_timestamp: Some(timestamp2), id: "span2".to_string(), @@ -860,28 +1059,28 @@ mod tests { "| sql_span1a | sql_test_span | 2023-02-01T15:30:00 |", "+------------+---------------+---------------------+", ], &verify_df); - // + let insert_sql = "INSERT INTO otel_logs_and_spans ( - project_id, timestamp, id, - parent_id, name, kind, - status_code, status_message, level, severity___severity_text, severity___severity_number, - body, duration, start_time, end_time - ) VALUES ( - 'test_project', TIMESTAMP '2023-01-01T10:00:00Z', 'sql_span1', - NULL, 'sql_test_span', NULL, - 'OK', 'span inserted successfully', 'INFO', 'INFORMATION', NULL, - NULL, 150000000, TIMESTAMP '2023-01-01T10:00:00Z', NULL - )"; + project_id, date, timestamp, id, hashes, + parent_id, name, kind, + status_code, status_message, level, severity___severity_text, severity___severity_number, + body, duration, start_time, end_time + ) VALUES ( + 'test_project', TIMESTAMP '2023-01-01', TIMESTAMP '2023-01-01T10:00:00Z', 'sql_span1', ARRAY[], + NULL, 'sql_test_span', NULL, + 'OK', 'span inserted successfully', 'INFO', 'INFORMATION', NULL, + NULL, 150000000, TIMESTAMP '2023-01-01T10:00:00Z', NULL + )"; let insert_result = ctx.sql(insert_sql).await?.collect().await?; #[rustfmt::skip] - assert_batches_eq!( - ["+-------+", - "| count |", - "+-------+", - "| 1 |", - "+-------+", - ], &insert_result); + assert_batches_eq!( + ["+-------+", + "| count |", + "+-------+", + "| 1 |", + "+-------+", + ], &insert_result); let verify_df = ctx .sql("SELECT project_id, id, name, timestamp, kind, status_code, severity___severity_text, duration, start_time from otel_logs_and_spans order by timestamp desc") @@ -902,7 +1101,7 @@ mod tests { log::info!("Inserting record directly via insert statement"); let insert_sql = "INSERT INTO otel_logs_and_spans ( - project_id, timestamp, observed_timestamp, id, + project_id, date, timestamp, observed_timestamp, id, hashes, parent_id, name, kind, status_code, status_message, level, severity___severity_text, severity___severity_number, body, duration, start_time, end_time, @@ -928,7 +1127,7 @@ mod tests { resource___service___version, resource___service___instance___id, resource___service___namespace, resource___telemetry___sdk___language, resource___telemetry___sdk___name, resource___telemetry___sdk___version, resource___user_agent___original ) VALUES ( - 'test_project', TIMESTAMP '2023-01-02T10:00:00Z', NULL, 'sql_span2', + 'test_project','2023-01-02', TIMESTAMP '2023-01-02T10:00:00Z', NULL, 'sql_span2', ARRAY[], NULL, 'sql_test_span', NULL, 'OK', 'span inserted successfully', 'INFO', NULL, NULL, NULL, 150000000, TIMESTAMP '2023-01-01T10:00:00Z', NULL, @@ -957,13 +1156,13 @@ mod tests { let insert_result = ctx.sql(insert_sql).await?.collect().await?; #[rustfmt::skip] - assert_batches_eq!( - ["+-------+", - "| count |", - "+-------+", - "| 1 |", - "+-------+", - ], &insert_result); + assert_batches_eq!( + ["+-------+", + "| count |", + "+-------+", + "| 1 |", + "+-------+", + ], &insert_result); // Verify that the SQL-inserted record exists let verify_df = ctx.sql("SELECT id, name, status_message FROM otel_logs_and_spans WHERE id = 'sql_span1'").await?; @@ -1023,23 +1222,23 @@ mod tests { // ], &insert_result); let verify_df = ctx - .sql("SELECT project_id, id, name, timestamp, kind, status_code, severity___severity_text, duration, start_time from otel_logs_and_spans order by timestamp desc") - .await? - .collect() - .await?; + .sql("SELECT project_id, id, name, timestamp, kind, status_code, severity___severity_text, duration, start_time from otel_logs_and_spans order by timestamp desc") + .await? + .collect() + .await?; #[rustfmt::skip] - assert_batches_eq!( - [ - "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", - "| project_id | id | name | timestamp | kind | status_code | severity___severity_text | duration | start_time |", - "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", - "| default | sql_span1a | sql_test_span | 2023-02-01T15:30:00 | | OK | | 150000000 | 2023-02-01T15:30:00 |", - "| test_project | sql_span2 | sql_test_span | 2023-01-02T10:00:00 | | OK | | 150000000 | 2023-01-01T10:00:00 |", - "| test_project | sql_span1 | sql_test_span | 2023-01-01T10:00:00 | | OK | INFORMATION | 150000000 | 2023-01-01T10:00:00 |", - "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", - ], - &verify_df - ); + assert_batches_eq!( + [ + "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", + "| project_id | id | name | timestamp | kind | status_code | severity___severity_text | duration | start_time |", + "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", + "| default | sql_span1a | sql_test_span | 2023-02-01T15:30:00 | | OK | | 150000000 | 2023-02-01T15:30:00 |", + "| test_project | sql_span2 | sql_test_span | 2023-01-02T10:00:00 | | OK | | 150000000 | 2023-01-01T10:00:00 |", + "| test_project | sql_span1 | sql_test_span | 2023-01-01T10:00:00 | | OK | INFORMATION | 150000000 | 2023-01-01T10:00:00 |", + "+--------------+------------+---------------+---------------------+------+-------------+--------------------------+-----------+---------------------+", + ], + &verify_df + ); Ok(()) } diff --git a/src/delta.rs b/src/delta.rs new file mode 100644 index 00000000..02e57fdf --- /dev/null +++ b/src/delta.rs @@ -0,0 +1,339 @@ +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use deltalake::{ + DeltaTable, DeltaTableBuilder, DeltaTableError, + arrow::record_batch::RecordBatch, + operations::write::WriteBuilder, + logstore::{default_logstore, ObjectStoreRef}, + operations::create::{CreateBuilder}, + kernel::StructField, +}; +use object_store::{aws::AmazonS3Builder, azure::MicrosoftAzureBuilder, gcp::GoogleCloudStorageBuilder, local::LocalFileSystem, memory::InMemory}; +use tokio; +use url::Url; + +use crate::store::{DeltaCacheBuilder, DeltaCacheConfig}; + +/// Helper struct for creating Delta tables with caching +pub struct CachedDeltaTableBuilder { + table_uri: String, + cache_config: Option, + storage_options: HashMap, +} + +impl CachedDeltaTableBuilder { + pub fn new>(table_uri: S) -> Self { + Self { + table_uri: table_uri.into(), + cache_config: None, + storage_options: HashMap::new(), + } + } + + /// Enable caching with custom configuration + pub fn with_cache_config(mut self, config: DeltaCacheConfig) -> Self { + self.cache_config = Some(config); + self + } + + /// Enable caching with default configuration + pub fn with_cache(mut self) -> Self { + self.cache_config = Some(DeltaCacheConfig::default()); + self + } + + /// Add storage options (AWS credentials, etc.) + pub fn with_storage_options(mut self, options: HashMap) -> Self { + self.storage_options = options; + self + } + + /// Add a single storage option + pub fn with_storage_option, V: Into>(mut self, key: K, value: V) -> Self { + self.storage_options.insert(key.into(), value.into()); + self + } + + /// Build the Delta table with caching + pub async fn build(&self) -> Result { + let base_store = self.create_base_object_store().await?; + + let final_store: ObjectStoreRef = if let Some(cache_config) = &self.cache_config { + // Wrap with cache + + + (DeltaCacheBuilder::new() + .with_memory_capacity(cache_config.memory_capacity) + .with_disk_capacity(cache_config.disk_capacity) + .with_disk_path(cache_config.disk_cache_dir.clone()) + .with_ttl(Duration::from_secs(cache_config.ttl_seconds)) + .with_compression(cache_config.compression_level) + .enable_metrics(cache_config.enable_metrics) + .enable_cache_warming(cache_config.enable_cache_warming) + .cache_transaction_logs(cache_config.cache_transaction_logs) + .cache_parquet_metadata(cache_config.cache_parquet_metadata) + .cache_checkpoints(cache_config.cache_checkpoints) + .build(base_store) + .await + .map_err(|e| DeltaTableError::ObjectStore { source: e })?) as _ + } else { + base_store + }; + + // Create Delta table with the (potentially cached) object store + DeltaTableBuilder::from_uri(&self.table_uri) + .with_storage_backend(final_store, Url::parse(&self.table_uri).unwrap()) + .load() + .await + } + + /// Create the base object store based on URI scheme + async fn create_base_object_store(&self) -> Result { + let uri = Url::parse(&self.table_uri).map_err(|e| DeltaTableError::Generic(format!("Invalid URI: {e}")))?; + + match uri.scheme() { + "s3" | "s3a" => { + let mut builder = AmazonS3Builder::new(); + + // Apply storage options + for (key, value) in &self.storage_options { + match key.as_str() { + "AWS_ACCESS_KEY_ID" => builder = builder.with_access_key_id(value), + "AWS_SECRET_ACCESS_KEY" => builder = builder.with_secret_access_key(value), + "AWS_REGION" => builder = builder.with_region(value), + "AWS_ENDPOINT" => builder = builder.with_endpoint(value), + "AWS_BUCKET_NAME" => builder = builder.with_bucket_name(value), + _ => {} + } + } + + let bucket = uri.host_str().ok_or_else(|| DeltaTableError::Generic("No bucket in S3 URI".to_string()))?; + + Ok(Arc::new( + builder.with_bucket_name(bucket).build().map_err(|e| DeltaTableError::ObjectStore { source: e })?, + )) + } + + "abfs" | "abfss" => { + let mut builder = MicrosoftAzureBuilder::new(); + + // Apply storage options + for (key, value) in &self.storage_options { + match key.as_str() { + "AZURE_STORAGE_ACCOUNT_NAME" => builder = builder.with_account(value), + "AZURE_STORAGE_ACCOUNT_KEY" => builder = builder.with_access_key(value), + // "AZURE_STORAGE_SAS_TOKEN" => builder = builder.with_sas_authorization(vec![(value,value)]), + "AZURE_STORAGE_CONTAINER_NAME" => builder = builder.with_container_name(value), + _ => {} + } + } + + Ok(Arc::new(builder.build().map_err(|e| DeltaTableError::ObjectStore { source: e })?)) + } + + "gs" => { + let mut builder = GoogleCloudStorageBuilder::new(); + + // Apply storage options + for (key, value) in &self.storage_options { + match key.as_str() { + "GOOGLE_SERVICE_ACCOUNT" => builder = builder.with_service_account_path(value), + "GOOGLE_SERVICE_ACCOUNT_KEY" => builder = builder.with_service_account_key(value), + "GOOGLE_BUCKET_NAME" => builder = builder.with_bucket_name(value), + _ => {} + } + } + + let bucket = uri.host_str().ok_or_else(|| DeltaTableError::Generic("No bucket in GCS URI".to_string()))?; + + Ok(Arc::new( + builder.with_bucket_name(bucket).build().map_err(|e| DeltaTableError::ObjectStore { source: e })?, + )) + } + + "file" => { + let path = uri.to_file_path().map_err(|_| DeltaTableError::Generic("Invalid file path".to_string()))?; + Ok(Arc::new( + LocalFileSystem::new_with_prefix(path).map_err(|e| DeltaTableError::ObjectStore { source: e })?, + )) + } + + "memory" => Ok(Arc::new(InMemory::new())), + + scheme => Err(DeltaTableError::Generic(format!("Unsupported scheme: {scheme}"))), + } + } +} + +/// Convenience functions for common Delta operations with caching +pub struct CachedDeltaOps; + +impl CachedDeltaOps { + /// Create a new Delta table with caching enabled + pub async fn create_table( + table_uri: &str, + schema: Arc, + cache_config: Option, + ) -> Result { + let mut builder = CachedDeltaTableBuilder::new(table_uri); + + if let Some(config) = cache_config { + builder = builder.with_cache_config(config); + } + + let store = builder.create_base_object_store().await?; + let url = Url::parse(table_uri).map_err(|e| DeltaTableError::Generic(e.to_string()))?; + let log_store = default_logstore(store, &url, &Default::default()); + + let columns: Vec = schema + .fields() + .iter() + .map(|f| StructField::try_from(f.as_ref()).unwrap()) + .collect(); + + CreateBuilder::new() + .with_log_store(log_store) + .with_table_name(table_uri) + .with_columns(columns) + .await + } + + /// Open an existing Delta table with caching + pub async fn open_table( + table_uri: &str, cache_config: Option, storage_options: Option>, + ) -> Result { + let mut builder = CachedDeltaTableBuilder::new(table_uri); + + if let Some(config) = cache_config { + builder = builder.with_cache_config(config); + } + + if let Some(options) = storage_options { + builder = builder.with_storage_options(options); + } + + builder.build().await + } + + /// Write data to a Delta table with caching + pub async fn write_to_table(table: &mut DeltaTable, batches: Vec) -> Result<(), DeltaTableError> { + let m = table.state.clone(); + WriteBuilder::new(table.log_store(), m).with_input_batches(batches); + + // Reload the table to see the new data + + Ok(table.load().await.unwrap()) + } +} + + + + + +#[cfg(test)] +mod tests { + use std::{fmt::Error, sync::Arc}; + + use arrow::{ + array::{Int32Array, StringArray}, + datatypes::{DataType, Field, Schema}, + record_batch::RecordBatch, + }; + use tempfile::{tempdir, TempDir}; + + use super::*; + + #[tokio::test] + async fn test_cached_delta_table_creation() { + let dir = tempdir().unwrap(); + let table_uri = format!("file://{}/", dir.path().to_str().unwrap()); + + let _schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ])); + + let cache_dir = dir.path().join("cache"); + std::fs::create_dir(&cache_dir).unwrap(); + + let cache_config = DeltaCacheConfig { + disk_cache_dir: cache_dir.to_str().unwrap().to_string(), + disk_capacity: 0, // Set to 0 to avoid using disk, but provide a valid path + ..Default::default() + }; + + // Create the table + let result = CachedDeltaOps::create_table(&table_uri, _schema.clone(), Some(cache_config.clone())).await; + + let table = result.unwrap(); + + assert_eq!( format!("file://{}", table.table_uri()), table_uri); + } + + #[tokio::test] + async fn test_write_and_read_with_cache() { + let temp_dir = TempDir::new().unwrap(); + let table_uri = format!("file://{}/", temp_dir.path().to_str().unwrap()); + + // Create schema + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ])); + + // Create table with cache + let mut table = CachedDeltaOps::create_table(&table_uri, schema.clone(), Some(DeltaCacheConfig::default())) + .await + .unwrap(); + + // Create some test data + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3])), Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"]))], + ) + .unwrap(); + + // Write data + CachedDeltaOps::write_to_table(&mut table, vec![batch]).await.unwrap(); + + // Read data back (should hit cache on subsequent reads) + let files = table.get_file_uris(); + assert!(files.is_ok()); + + + + // Read again to test cache hit + let _files_again = table.get_file_uris(); + + + } + + #[tokio::test] + async fn test_write_and_read_with_caches3() -> Result<(),deltalake::DeltaTableError> { + // Example 1: Simple cached Delta table + let cache_config = DeltaCacheConfig { + memory_capacity: 256 * 1024 * 1024, // 256MB + disk_capacity: 0, // Use memory-only cache + disk_cache_dir: "".to_string(), // Empty string for memory-only + ttl_seconds: 3600, // 1 hour + enable_metrics: true, + enable_cache_warming: true, + ..Default::default() + }; + + let table = CachedDeltaTableBuilder::new("s3://my-bucket/my-table") + .with_cache_config(cache_config) + .with_storage_option("AWS_REGION", "us-west-2") + .with_storage_option("AWS_ACCESS_KEY_ID", "your-access-key") + .with_storage_option("AWS_SECRET_ACCESS_KEY", "your-secret-key") + .build() + .await?; + + println!("Table loaded with {} files", table.get_files_count()); + + + + + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 149debdb..f579cf1a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,6 @@ // lib.rs - Export modules for use in tests +pub mod batch_queue; pub mod database; +pub mod delta; +pub mod obj_store; pub mod persistent_queue; diff --git a/src/main.rs b/src/main.rs index 084108b0..3971aa1f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,13 +1,16 @@ // main.rs +mod batch_queue; mod database; mod persistent_queue; -use actix_web::{middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder}; +use std::{env, sync::Arc}; + +use actix_web::{App, HttpResponse, HttpServer, Responder, middleware::Logger, post, web}; +use batch_queue::BatchQueue; use database::Database; use dotenv::dotenv; use futures::TryFutureExt; use serde::Deserialize; -use std::{env, sync::Arc}; -use tokio::time::{sleep, Duration}; +use tokio::time::{Duration, sleep}; use tokio_util::sync::CancellationToken; use tracing::{error, info}; use tracing_subscriber::EnvFilter; @@ -18,10 +21,10 @@ struct AppInfo {} #[derive(Deserialize)] struct RegisterProjectRequest { project_id: String, - bucket: String, + bucket: String, access_key: String, secret_key: String, - endpoint: Option, + endpoint: Option, } #[post("/register_project")] @@ -54,15 +57,29 @@ async fn main() -> anyhow::Result<()> { info!("Starting TimeFusion application"); // Initialize database - let db = Database::new().await?; + let mut db = Database::new().await?; info!("Database initialized successfully"); - // Create and setup session context + // Setup batch processing with configurable params + let interval_ms = env::var("BATCH_INTERVAL_MS").ok().and_then(|v| v.parse().ok()).unwrap_or(1000); + let max_size = env::var("MAX_BATCH_SIZE").ok().and_then(|v| v.parse().ok()).unwrap_or(1000); + let enable_queue = env::var("ENABLE_BATCH_QUEUE").unwrap_or_else(|_| "false".to_string()) == "true"; + + // Create batch queue + let batch_queue = Arc::new(BatchQueue::new(Arc::new(db.clone()), interval_ms, max_size)); + info!( + "Batch queue configured (enabled={}, interval={}ms, max_size={})", + enable_queue, interval_ms, max_size + ); + + // Apply and setup + db = db.with_batch_queue(Arc::clone(&batch_queue)); + // Start maintenance schedulers for regular optimize and vacuum + db = db.start_maintenance_schedulers().await?; let session_context = db.create_session_context(); db.setup_session_context(&session_context)?; - info!("Session context setup complete"); - // Wrap database in Arc for sharing + // Wrap for sharing let db = Arc::new(db); let app_info = web::Data::new(AppInfo {}); @@ -132,7 +149,10 @@ async fn main() -> anyhow::Result<()> { _ = pg_server.map_err(|e| error!("PGWire server task failed: {:?}", e)) => {}, _ = http_task.map_err(|e| error!("HTTP server task failed: {:?}", e)) => {}, _ = tokio::signal::ctrl_c() => { - info!("Received Ctrl+C, initiating shutdown."); + info!("Received Ctrl+C, initiating shutdown"); + + // Shutdown in order: batch queue first to flush pending data + batch_queue.shutdown().await; shutdown_token.cancel(); http_server_handle.stop(true).await; sleep(Duration::from_secs(1)).await; diff --git a/src/obj_store.rs b/src/obj_store.rs new file mode 100644 index 00000000..5b7eb76c --- /dev/null +++ b/src/obj_store.rs @@ -0,0 +1,663 @@ +use std::{ + collections::HashMap, + fmt::Debug, + ops::Range, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use async_stream::stream; +use async_trait::async_trait; +use bytes::Bytes; +use chrono::DateTime; +use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; +use futures::stream::{BoxStream, StreamExt}; +use object_store::{ + GetOptions, GetRange, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, PutOptions, PutPayload, + PutResult, Result as ObjectStoreResult, path::Path, +}; +use serde::{Deserialize, Serialize}; +use tokio::sync::RwLock; +use tracing::{debug, error, info, warn}; + +/// Configuration for the Delta cache +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeltaCacheConfig { + /// Memory cache capacity in bytes + pub memory_capacity: usize, + /// Disk cache capacity in bytes + pub disk_capacity: usize, + /// Disk cache directory path + pub disk_cache_dir: String, + /// TTL for cached objects in seconds + pub ttl_seconds: u64, + /// Whether to cache transaction logs + pub cache_transaction_logs: bool, + /// Whether to cache parquet metadata + pub cache_parquet_metadata: bool, + /// Whether to cache checkpoint files + pub cache_checkpoints: bool, + /// Maximum object size to cache (in bytes) + pub max_object_size: usize, + /// Enable metrics collection + pub enable_metrics: bool, + /// Cache warming on startup + pub enable_cache_warming: bool, + /// Compression level for cached data (0-9, 0=no compression) + pub compression_level: u8, +} + +impl Default for DeltaCacheConfig { + fn default() -> Self { + Self { + memory_capacity: 256 * 1024 * 1024, // 256MB + disk_capacity: 1024 * 1024 * 1024, // 1GB + disk_cache_dir: "/tmp/delta_cache".to_string(), + ttl_seconds: 3600, // 1 hour + cache_transaction_logs: true, + cache_parquet_metadata: true, + cache_checkpoints: true, + max_object_size: 10 * 1024 * 1024, // 10MB max + enable_metrics: true, + enable_cache_warming: false, + compression_level: 3, // Light compression by default + } + } +} + +/// Cache metrics for monitoring +#[derive(Debug, Default, Clone)] +pub struct CacheMetrics { + pub hits: u64, + pub misses: u64, + pub evictions: u64, + pub errors: u64, + pub total_requests: u64, + pub cache_size_bytes: u64, +} + +impl CacheMetrics { + pub fn hit_rate(&self) -> f64 { + if self.total_requests == 0 { 0.0 } else { self.hits as f64 / self.total_requests as f64 } + } +} + +/// cached object with metadata and compression +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CachedObject { + data: Vec, + cached_at: u64, + original_size: usize, + is_compressed: bool, + etag: Option, + last_modified: Option, +} + +impl CachedObject { + fn new(data: Vec, meta: &ObjectMeta, compression_level: u8) -> Self { + let original_size = data.len(); + let (final_data, is_compressed) = if compression_level > 0 && data.len() > 1024 { + match Self::compress(&data, compression_level) { + Ok(compressed) if compressed.len() < data.len() => (compressed, true), + _ => (data, false), + } + } else { + (data, false) + }; + + Self { + data: final_data, + cached_at: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(), + original_size, + is_compressed, + etag: meta.e_tag.clone(), + last_modified: Some(meta.last_modified.timestamp()), + } + } + + fn get_data(&self) -> Result, std::io::Error> { + if self.is_compressed { Self::decompress(&self.data) } else { Ok(self.data.clone()) } + } + + fn compress(data: &[u8], level: u8) -> Result, std::io::Error> { + use std::io::Write; + + use flate2::{Compression, write::GzEncoder}; + + let mut encoder = GzEncoder::new(Vec::new(), Compression::new(level as u32)); + encoder.write_all(data)?; + encoder.finish() + } + + fn decompress(data: &[u8]) -> Result, std::io::Error> { + use std::io::Read; + + use flate2::read::GzDecoder; + + let mut decoder = GzDecoder::new(data); + let mut decompressed = Vec::new(); + decoder.read_to_end(&mut decompressed)?; + Ok(decompressed) + } + + fn is_valid(&self, ttl_seconds: u64) -> bool { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + (now - self.cached_at) < ttl_seconds + } + + fn matches_meta(&self, meta: &ObjectMeta) -> bool { + // Check ETag if available + if let (Some(cached_etag), Some(meta_etag)) = (&self.etag, &meta.e_tag) { + return cached_etag == meta_etag; + } + + // Fallback to last modified time + if let (Some(cached_modified), meta_modified) = (self.last_modified, meta.last_modified) { + return cached_modified == meta_modified.timestamp(); + } + + // If no metadata available, assume valid (will rely on TTL) + true + } +} + +/// Delta-optimized object store cache wrapper +#[derive(Debug)] +pub struct DeltaCachedObjectStore { + inner: Arc, + cache: Arc>, + config: DeltaCacheConfig, + metrics: Arc>, + // Track frequently accessed paths for cache warming + access_patterns: Arc>>, +} + +impl DeltaCachedObjectStore { + /// Create a new cached object store + pub async fn new(inner: Arc, config: DeltaCacheConfig) -> ObjectStoreResult { + // Build the hybrid cache + let mut builder = HybridCacheBuilder::new() + .memory(config.memory_capacity) + .storage(Engine::Large); + + if config.disk_capacity > 0 { + builder = builder.with_device_options( + DirectFsDeviceOptions::new(&config.disk_cache_dir) + .with_capacity(config.disk_capacity), + ); + } + + let cache = builder.build() + .await + .map_err(|e| object_store::Error::Generic { + store: "DeltaCache", + source: Box::new(std::io::Error::other(e)), + })?; + + info!( + "Initialized Delta cache: memory={}MB, disk={}GB, dir={}, compression={}", + config.memory_capacity / (1024 * 1024), + config.disk_capacity / (1024 * 1024 * 1024), + config.disk_cache_dir, + if config.compression_level > 0 { "enabled" } else { "disabled" } + ); + + let store = Self { + inner, + cache: Arc::new(cache), + config, + metrics: Arc::new(RwLock::new(CacheMetrics::default())), + access_patterns: Arc::new(RwLock::new(HashMap::new())), + }; + + // Optionally warm the cache + if store.config.enable_cache_warming { + tokio::spawn({ + let store_clone = store.clone(); + async move { + if let Err(e) = store_clone.warm_cache().await { + warn!("Cache warming failed: {}", e); + } + } + }); + } + + Ok(store) + } + + /// Get current cache metrics + pub async fn metrics(&self) -> CacheMetrics { + self.metrics.read().await.clone() + } + + /// Warm the cache by pre-loading frequently accessed files + async fn warm_cache(&self) -> ObjectStoreResult<()> { + info!("Starting cache warming..."); + + // Focus on Delta log directory + let delta_log_prefix = Path::from("_delta_log"); + let mut stream = self.inner.list(Some(&delta_log_prefix)); + let mut warmed_count = 0; + + while let Some(meta_result) = futures::StreamExt::next(&mut stream).await { + let meta = meta_result?; + + // Only warm small, frequently accessed files + if meta.size <= (1024 * 1024) && self.should_cache(&meta.location) { + match self.inner.get(&meta.location).await { + Ok(result) => { + if let Ok(bytes) = result.bytes().await { + let cache_key = self.make_cache_key(&meta.location, None); + let cached_obj = CachedObject::new(bytes.to_vec(), &meta, self.config.compression_level); + + if self.cache.insert(cache_key, cached_obj).get_data().is_ok() { + warmed_count += 1; + } + } + } + Err(e) => debug!("Failed to warm cache for {}: {}", meta.location, e), + } + } + } + + info!("Cache warming completed: {} files preloaded", warmed_count); + Ok(()) + } + + /// Enhanced path caching logic with more granular control + fn should_cache(&self, path: &Path) -> bool { + let path_str = path.as_ref(); + + // Always cache Delta log directory contents + if path_str.contains("_delta_log/") { + // Transaction logs + if path_str.ends_with(".json") && self.config.cache_transaction_logs { + return true; + } + // Checkpoint files + if path_str.ends_with(".checkpoint.parquet") && self.config.cache_checkpoints { + return true; + } + // Other Delta log files (like .crc files) + return true; + } + + // Parquet metadata files + if self.config.cache_parquet_metadata && (path_str.ends_with(".parquet") || path_str.contains("_metadata") || path_str.ends_with("_common_metadata")) { + return true; + } + + false + } + + /// Create cache key from path and optional range + fn make_cache_key(&self, path: &Path, range: Option<&GetRange>) -> String { + match range { + Some(GetRange::Bounded(r)) => format!("{}:{}:{}", path.as_ref(), r.start, r.end), + Some(GetRange::Offset(offset)) => format!("{}:{}:", path.as_ref(), offset), + Some(GetRange::Suffix(suffix)) => format!("{}:suffix:{}", path.as_ref(), suffix), + None => path.as_ref().to_string(), + } + } + + /// Update access patterns for analytics + async fn record_access(&self, path: &str) { + let mut patterns = self.access_patterns.write().await; + *patterns.entry(path.to_string()).or_insert(0) += 1; + } + + /// Update metrics + async fn update_metrics(&self, update_fn: F) + where + F: FnOnce(&mut CacheMetrics), + { + if self.config.enable_metrics { + let mut metrics = self.metrics.write().await; + update_fn(&mut metrics); + } + } + + /// Enhanced cache retrieval with metadata validation + async fn get_with_cache(&self, location: &Path, options: GetOptions) -> ObjectStoreResult { + self.update_metrics(|m| m.total_requests += 1).await; + + // Check if we should cache this path + if !self.should_cache(location) { + debug!("Path not cacheable, delegating: {}", location); + return self.inner.get_opts(location, options).await; + } + + // Record access pattern + self.record_access(location.as_ref()).await; + + // For range requests, bypass cache for now (could be enhanced later) + if options.range.is_some() { + debug!("Range request, bypassing cache: {}", location); + return self.inner.get_opts(location, options).await; + } + + let cache_key = self.make_cache_key(location, None); + + // Try to get from cache first + if let Ok(Some(cached_entry)) = self.cache.get(&cache_key).await { + let cached_obj = cached_entry.value(); + + // Check if cache entry is still valid + if cached_obj.is_valid(self.config.ttl_seconds) { + // Get fresh metadata for validation + match self.inner.head(location).await { + Ok(meta) => { + // Validate cache against metadata + if cached_obj.matches_meta(&meta) { + debug!("Cache hit for: {}", location); + self.update_metrics(|m| m.hits += 1).await; + + // Decompress if needed + match cached_obj.get_data() { + Ok(data) => { + let bytes = Bytes::from(data); + return Ok(GetResult { + payload: GetResultPayload::Stream(Box::pin(futures::stream::once(async { Ok(bytes) }))), + meta: meta.clone(), + range: 0..meta.size, + attributes: Default::default(), + }); + } + Err(e) => { + error!("Failed to decompress cached data for {}: {}", location, e); + // Fall through to cache miss + } + } + } else { + debug!("Cache invalidated due to metadata mismatch: {}", location); + // Remove stale entry + self.cache.remove(&cache_key); + } + } + Err(e) => { + debug!("Failed to get metadata for cache validation: {} - {}", location, e); + // Use cached data anyway if metadata lookup fails + if let Ok(data) = cached_obj.get_data() { + self.update_metrics(|m| m.hits += 1).await; + let bytes = Bytes::from(data); + return Ok(GetResult { + payload: GetResultPayload::Stream(Box::pin(futures::stream::once(async { Ok(bytes) }))), + meta: ObjectMeta { + location: location.clone(), + last_modified: DateTime::::MIN_UTC, + size: cached_obj.original_size as u64, + e_tag: cached_obj.etag.clone(), + version: None, + }, + range: 0..cached_obj.original_size as u64, + attributes: Default::default(), + }); + } + } + } + } else { + debug!("Cache entry expired: {}", location); + self.cache.remove(&cache_key); + } + } + + // Cache miss - fetch from underlying store + debug!("Cache miss, fetching: {}", location); + self.update_metrics(|m| m.misses += 1).await; + + let result = self.inner.get_opts(location, options.clone()).await?; + let meta = result.meta.clone(); + + // Only cache if object size is within limits + if meta.size <= self.config.max_object_size as u64 { + // Read the entire payload for caching + let bytes = result.bytes().await?; + + // Create cached object with compression + let cached_obj = CachedObject::new(bytes.to_vec(), &meta, self.config.compression_level); + + // Insert into cache asynchronously + let cache_clone = self.cache.clone(); + let key_clone = cache_key.clone(); + tokio::spawn(async move { + if let Err(e) = cache_clone.insert(key_clone, cached_obj).get_data() { + debug!("Failed to insert into cache: {}", e); + } + }); + + debug!("Cached object: {} (size: {} bytes)", location, bytes.len()); + + // Return the data + Ok(GetResult { + payload: GetResultPayload::Stream(Box::pin(futures::stream::once(async { Ok(bytes) }))), + meta: meta.clone(), + range: 0..meta.size, + attributes: Default::default(), + }) + } else { + warn!("Object too large to cache: {} ({} bytes)", location, meta.size); + Ok(result) + } + } + + /// Get access patterns for analytics + pub async fn get_access_patterns(&self) -> HashMap { + self.access_patterns.read().await.clone() + } + + /// Clear access patterns + pub async fn clear_access_patterns(&self) { + self.access_patterns.write().await.clear(); + } +} + +// Clone implementation for the store +impl Clone for DeltaCachedObjectStore { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + cache: self.cache.clone(), + config: self.config.clone(), + metrics: self.metrics.clone(), + access_patterns: self.access_patterns.clone(), + } + } +} + +#[async_trait] +impl ObjectStore for DeltaCachedObjectStore { + async fn put(&self, location: &Path, payload: PutPayload) -> ObjectStoreResult { + let result = self.inner.put(location, payload).await?; + + // Invalidate cache for this path on writes + if self.should_cache(location) { + let cache_key = self.make_cache_key(location, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache for: {}", location); + } + + Ok(result) + } + + async fn put_opts(&self, location: &Path, payload: PutPayload, opts: PutOptions) -> ObjectStoreResult { + let result = self.inner.put_opts(location, payload, opts).await?; + + // Invalidate cache for this path on writes + if self.should_cache(location) { + let cache_key = self.make_cache_key(location, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache for: {}", location); + } + + Ok(result) + } + + async fn put_multipart(&self, location: &Path) -> ObjectStoreResult> { + self.inner.put_multipart(location).await + } + + async fn put_multipart_opts(&self, location: &Path, opts: PutMultipartOpts) -> ObjectStoreResult> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get(&self, location: &Path) -> ObjectStoreResult { + self.get_with_cache(location, GetOptions::default()).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> ObjectStoreResult { + self.get_with_cache(location, options).await + } + + async fn get_range(&self, location: &Path, range: Range) -> ObjectStoreResult { + let options = GetOptions { + range: Some(GetRange::Bounded(range)), + ..Default::default() + }; + let result = self.get_with_cache(location, options).await?; + result.bytes().await + } + + async fn head(&self, location: &Path) -> ObjectStoreResult { + // For metadata requests, always fetch fresh data + self.inner.head(location).await + } + + async fn delete(&self, location: &Path) -> ObjectStoreResult<()> { + let result = self.inner.delete(location).await; + + // Invalidate cache on delete + if self.should_cache(location) { + let cache_key = self.make_cache_key(location, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache on delete: {}", location); + } + + result + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, ObjectStoreResult> { + let inner = self.inner.clone(); + let prefix = prefix.map(|p| p.to_owned()); + Box::pin(stream! { + let mut stream = inner.list(prefix.as_ref()); + while let Some(item) = stream.next().await { + yield item; + } + }) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> ObjectStoreResult { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { + let result = self.inner.copy(from, to).await; + + // Invalidate cache for destination + if self.should_cache(to) { + let cache_key = self.make_cache_key(to, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache on copy destination: {}", to); + } + + result + } + + async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> ObjectStoreResult<()> { + let result = self.inner.copy_if_not_exists(from, to).await; + + // Invalidate cache for destination + if self.should_cache(to) { + let cache_key = self.make_cache_key(to, None); + let _ = self.cache.remove(&cache_key); + debug!("Invalidated cache on conditional copy: {}", to); + } + + result + } +} + +impl std::fmt::Display for DeltaCachedObjectStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "DeltaCached({})", self.inner) + } +} + +/// Enhanced builder with more configuration options +pub struct DeltaCacheBuilder { + config: DeltaCacheConfig, +} + +impl DeltaCacheBuilder { + pub fn new() -> Self { + Self { + config: DeltaCacheConfig::default(), + } + } + + pub fn with_memory_capacity(mut self, capacity: usize) -> Self { + self.config.memory_capacity = capacity; + self + } + + pub fn with_disk_capacity(mut self, capacity: usize) -> Self { + self.config.disk_capacity = capacity; + self + } + + pub fn with_disk_path>(mut self, path: P) -> Self { + self.config.disk_cache_dir = path.into(); + self + } + + pub fn with_ttl(mut self, ttl: Duration) -> Self { + self.config.ttl_seconds = ttl.as_secs(); + self + } + + pub fn with_max_object_size(mut self, size: usize) -> Self { + self.config.max_object_size = size; + self + } + + pub fn with_compression(mut self, level: u8) -> Self { + self.config.compression_level = level.min(9); + self + } + + pub fn enable_metrics(mut self, enable: bool) -> Self { + self.config.enable_metrics = enable; + self + } + + pub fn enable_cache_warming(mut self, enable: bool) -> Self { + self.config.enable_cache_warming = enable; + self + } + + pub fn cache_transaction_logs(mut self, enable: bool) -> Self { + self.config.cache_transaction_logs = enable; + self + } + + pub fn cache_parquet_metadata(mut self, enable: bool) -> Self { + self.config.cache_parquet_metadata = enable; + self + } + + pub fn cache_checkpoints(mut self, enable: bool) -> Self { + self.config.cache_checkpoints = enable; + self + } + + pub async fn build(self, inner: Arc) -> ObjectStoreResult> { + let cached_store = DeltaCachedObjectStore::new(inner, self.config).await?; + Ok(Arc::new(cached_store)) + } +} + +impl Default for DeltaCacheBuilder { + fn default() -> Self { + Self::new() + } +} diff --git a/src/persistent_queue.rs b/src/persistent_queue.rs index 331cdff0..c8d00466 100644 --- a/src/persistent_queue.rs +++ b/src/persistent_queue.rs @@ -1,25 +1,29 @@ -use std::sync::Arc; - -use arrow_schema::{DataType, TimeUnit}; -use arrow_schema::{Field, Schema, SchemaRef}; -use delta_kernel::schema::StructField; -use serde::{Deserialize, Serialize}; -use serde_arrow::schema::SchemaLike; -use serde_arrow::schema::TracingOptions; +use std::{str::FromStr, sync::Arc}; + +use arrow_schema::{DataType, Field, FieldRef, Schema, SchemaRef}; +use delta_kernel::{parquet::format::SortingColumn, schema::StructField}; +use log::debug; +use serde::{Deserialize, Deserializer, Serialize, de::Error as DeError}; +use serde_arrow::schema::{SchemaLike, TracingOptions}; use serde_json::json; +use serde_with::serde_as; #[allow(non_snake_case)] +#[serde_as] #[derive(Serialize, Deserialize, Clone, Default)] pub struct OtelLogsAndSpans { + #[serde(with = "chrono::serde::ts_microseconds")] + pub timestamp: chrono::DateTime, + #[serde(with = "chrono::serde::ts_microseconds_option")] pub observed_timestamp: Option>, - pub id: String, - pub parent_id: Option, - pub hashes: Vec, // all relevant hashes can be stored here for item identification - pub name: Option, - pub kind: Option, // logs, span, request - pub status_code: Option, + pub id: String, + pub parent_id: Option, + pub hashes: Vec, // all relevant hashes can be stored here for item identification + pub name: Option, + pub kind: Option, // logs, span, request + pub status_code: Option, pub status_message: Option, // Logs specific @@ -28,7 +32,7 @@ pub struct OtelLogsAndSpans { // Severity pub severity: Option, // severity as json - pub severity___severity_text: Option, + pub severity___severity_text: Option, pub severity___severity_number: Option, pub body: Option, // body as json json @@ -38,16 +42,16 @@ pub struct OtelLogsAndSpans { #[serde(with = "chrono::serde::ts_microseconds_option")] pub start_time: Option>, #[serde(with = "chrono::serde::ts_microseconds_option")] - pub end_time: Option>, + pub end_time: Option>, // Context - pub context: Option, // context as json + pub context: Option, // context as json // - pub context___trace_id: Option, - pub context___span_id: Option, + pub context___trace_id: Option, + pub context___span_id: Option, pub context___trace_state: Option, pub context___trace_flags: Option, - pub context___is_remote: Option, + pub context___is_remote: Option, // Events pub events: Option, // events json @@ -56,108 +60,118 @@ pub struct OtelLogsAndSpans { pub links: Option, // links json // Attributes - pub attributes: Option, // attirbutes object as json + pub attributes: Option, // attirbutes object as json // Server and client pub attributes___client___address: Option, - pub attributes___client___port: Option, + pub attributes___client___port: Option, pub attributes___server___address: Option, - pub attributes___server___port: Option, + pub attributes___server___port: Option, // network https://opentelemetry.io/docs/specs/semconv/attributes-registry/network/ - pub attributes___network___local__address: Option, - pub attributes___network___local__port: Option, - pub attributes___network___peer___address: Option, - pub attributes___network___peer__port: Option, - pub attributes___network___protocol___name: Option, + pub attributes___network___local__address: Option, + pub attributes___network___local__port: Option, + pub attributes___network___peer___address: Option, + pub attributes___network___peer__port: Option, + pub attributes___network___protocol___name: Option, pub attributes___network___protocol___version: Option, - pub attributes___network___transport: Option, - pub attributes___network___type: Option, + pub attributes___network___transport: Option, + pub attributes___network___type: Option, // Source Code Attributes - pub attributes___code___number: Option, - pub attributes___code___file___path: Option, + pub attributes___code___number: Option, + pub attributes___code___file___path: Option, pub attributes___code___function___name: Option, - pub attributes___code___line___number: Option, - pub attributes___code___stacktrace: Option, + pub attributes___code___line___number: Option, + pub attributes___code___stacktrace: Option, // Log records. https://opentelemetry.io/docs/specs/semconv/general/logs/ pub attributes___log__record___original: Option, - pub attributes___log__record___uid: Option, + pub attributes___log__record___uid: Option, // Exception https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/ - pub attributes___error___type: Option, - pub attributes___exception___type: Option, - pub attributes___exception___message: Option, + pub attributes___error___type: Option, + pub attributes___exception___type: Option, + pub attributes___exception___message: Option, pub attributes___exception___stacktrace: Option, // URL https://opentelemetry.io/docs/specs/semconv/attributes-registry/url/ pub attributes___url___fragment: Option, - pub attributes___url___full: Option, - pub attributes___url___path: Option, - pub attributes___url___query: Option, - pub attributes___url___scheme: Option, + pub attributes___url___full: Option, + pub attributes___url___path: Option, + pub attributes___url___query: Option, + pub attributes___url___scheme: Option, // Useragent https://opentelemetry.io/docs/specs/semconv/attributes-registry/user-agent/ pub attributes___user_agent___original: Option, // HTTP https://opentelemetry.io/docs/specs/semconv/http/http-spans/ - pub attributes___http___request___method: Option, + pub attributes___http___request___method: Option, pub attributes___http___request___method_original: Option, - pub attributes___http___response___status_code: Option, - pub attributes___http___request___resend_count: Option, - pub attributes___http___request___body___size: Option, + pub attributes___http___response___status_code: Option, + pub attributes___http___request___resend_count: Option, + pub attributes___http___request___body___size: Option, // Session https://opentelemetry.io/docs/specs/semconv/general/session/ - pub attributes___session___id: Option, + pub attributes___session___id: Option, pub attributes___session___previous___id: Option, // Database https://opentelemetry.io/docs/specs/semconv/database/database-spans/ - pub attributes___db___system___name: Option, - pub attributes___db___collection___name: Option, - pub attributes___db___namespace: Option, - pub attributes___db___operation___name: Option, - pub attributes___db___response___status_code: Option, + pub attributes___db___system___name: Option, + pub attributes___db___collection___name: Option, + pub attributes___db___namespace: Option, + pub attributes___db___operation___name: Option, + pub attributes___db___response___status_code: Option, pub attributes___db___operation___batch___size: Option, - pub attributes___db___query___summary: Option, - pub attributes___db___query___text: Option, + pub attributes___db___query___summary: Option, + pub attributes___db___query___text: Option, // https://opentelemetry.io/docs/specs/semconv/attributes-registry/user/ - pub attributes___user___id: Option, - pub attributes___user___email: Option, + pub attributes___user___id: Option, + pub attributes___user___email: Option, pub attributes___user___full_name: Option, - pub attributes___user___name: Option, - pub attributes___user___hash: Option, + pub attributes___user___name: Option, + pub attributes___user___hash: Option, // Resource pub resource: Option, // resource as json // Resource Attributes (subset) https://opentelemetry.io/docs/specs/semconv/resource/ - pub resource___service___name: Option, - pub resource___service___version: Option, + pub resource___service___name: Option, + pub resource___service___version: Option, pub resource___service___instance___id: Option, - pub resource___service___namespace: Option, + pub resource___service___namespace: Option, pub resource___telemetry___sdk___language: Option, - pub resource___telemetry___sdk___name: Option, - pub resource___telemetry___sdk___version: Option, + pub resource___telemetry___sdk___name: Option, + pub resource___telemetry___sdk___version: Option, pub resource___user_agent___original: Option, // Kept at the bottom to make delta-rs happy, so its schema matches datafusion. // Seems delta removes the partition ids from the normal schema and moves them to the end. // Top-level fields - pub project_id: String, + pub project_id: String, - #[serde(with = "chrono::serde::ts_microseconds")] - pub timestamp: chrono::DateTime, + #[serde(default)] + #[serde(deserialize_with = "default_on_empty_string")] + pub date: chrono::NaiveDate, } impl OtelLogsAndSpans { pub fn table_name() -> String { "otel_logs_and_spans".to_string() } - pub fn columns() -> anyhow::Result> { + + pub fn fields() -> anyhow::Result> { let tracing_options = TracingOptions::default() + .strings_as_large_utf8(false) + .sequence_as_large_list(false) + .sequence_as_large_list(false) .overwrite("project_id", json!({"name": "project_id", "data_type": "Utf8", "nullable": false}))? + .overwrite("date", json!({"name": "date", "data_type": "Date32", "nullable": false}))? + .overwrite("duration", json!({"name": "duration", "data_type": "UInt64", "nullable": true}))? + .overwrite("body", json!({"name":"body", "data_type": "Utf8", "nullable": true}))? + .overwrite("attributes", json!({"name":"attributes", "data_type": "Utf8", "nullable": true}))? + .overwrite("resource", json!({"name":"resource", "data_type": "Utf8", "nullable": true}))? .overwrite( "timestamp", json!({"name": "timestamp", "data_type": "Timestamp(Microsecond, None)", "nullable": false}), @@ -176,10 +190,15 @@ impl OtelLogsAndSpans { json!({"name": "end_time", "data_type": "Timestamp(Microsecond, None)", "nullable": true}), )?; - let fields = Vec::::from_type::(tracing_options)?; + Ok(Vec::::from_type::(tracing_options)?) + } + + pub fn columns() -> anyhow::Result> { + let fields = OtelLogsAndSpans::fields()?; let vec_refs: Vec = fields.iter().map(|arc_field| arc_field.as_ref().try_into().unwrap()).collect(); assert_eq!(fields[fields.len() - 2].data_type(), &DataType::Utf8); - assert_eq!(fields[fields.len() - 1].data_type(), &DataType::Timestamp(TimeUnit::Microsecond, None)); + assert_eq!(fields[fields.len() - 1].data_type(), &DataType::Date32); + debug!("schema_field columns {:?}", vec_refs); Ok(vec_refs) } @@ -195,6 +214,42 @@ impl OtelLogsAndSpans { } pub fn partitions() -> Vec { - vec!["project_id".to_string(), "timestamp".to_string()] + vec!["project_id".to_string(), "date".to_string()] + } + + pub fn sorting_columns() -> Vec { + // Define sorting columns for the parquet files to improve query performance + vec![ + SortingColumn { + column_idx: 0, // timestamp is likely first in the schema + descending: true, // newest first + nulls_first: false, + }, + SortingColumn { + column_idx: 3, // id + descending: false, + nulls_first: false, + }, + ] + } + + pub fn z_order_columns() -> Vec { + // Define z-order columns for efficient time-series range queries + vec!["timestamp".to_string()] + } +} + +pub fn default_on_empty_string<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: Deserialize<'de> + Default + FromStr, + ::Err: std::fmt::Display, +{ + let opt = Option::::deserialize(deserializer)?; + + match opt { + None => Ok(T::default()), + Some(s) if s.is_empty() => Ok(T::default()), + Some(s) => T::from_str(&s).map_err(DeError::custom), } } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index bace96c6..01d55516 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,13 +1,16 @@ #[cfg(test)] mod integration { + use std::{ + collections::HashSet, + sync::{Arc, Mutex}, + time::{Duration, Instant}, + }; + use anyhow::Result; use dotenv::dotenv; use rand::Rng; use scopeguard; use serial_test::serial; - use std::collections::HashSet; - use std::sync::{Arc, Mutex}; - use std::time::{Duration, Instant}; use timefusion::database::Database; use tokio::{sync::Notify, time::sleep}; use tokio_postgres::{Client, NoTls}; @@ -105,8 +108,9 @@ mod integration { // Insert test data let timestamp_str = format!("'{}'", chrono::Utc::now().format("%Y-%m-%d %H:%M:%S")); let insert_query = format!( - "INSERT INTO otel_logs_and_spans (project_id, timestamp, id, name, status_code, status_message, level) - VALUES ($1, {}, $2, $3, $4, $5, $6)", + "INSERT INTO otel_logs_and_spans (project_id, date, timestamp, id, name, status_code, status_message, level, hashes) + VALUES ($1, {}, {}, $2, $3, $4, $5, $6, ARRAY[])", + chrono::Utc::now().date_naive().to_string(), timestamp_str ); @@ -151,7 +155,7 @@ mod integration { assert_eq!(count_rows[0].get::<_, String>(0), "test_project", "project_id should match"); let count_rows = client.query("SELECT * FROM otel_logs_and_spans WHERE project_id = $1", &[&"test_project"]).await?; - assert_eq!(count_rows[0].columns().len(), 84, "Should return all 84 columns"); + assert_eq!(count_rows[0].columns().len(), 86, "Should return all 84 columns"); Ok::<_, tokio_postgres::Error>(()) } @@ -190,8 +194,9 @@ mod integration { // Create timestamp for the insert query let timestamp_str = format!("'{}'", chrono::Utc::now().format("%Y-%m-%d %H:%M:%S")); let insert_query = format!( - "INSERT INTO otel_logs_and_spans (project_id, timestamp, id, name, status_code, status_message, level) - VALUES ($1, {}, $2, $3, $4, $5, $6)", + "INSERT INTO otel_logs_and_spans (project_id, date, timestamp, id, name, status_code, status_message, level, hashes) + VALUES ($1, {}, {}, $2, $3, $4, $5, $6, ARRAY[])", + chrono::Utc::now().date_naive().to_string(), timestamp_str ); diff --git a/tests/sqllogictest.rs b/tests/sqllogictest.rs index e7832e1f..ebade9d4 100644 --- a/tests/sqllogictest.rs +++ b/tests/sqllogictest.rs @@ -1,15 +1,16 @@ #[cfg(test)] mod sqllogictest_tests { - use anyhow::Result; - use async_trait::async_trait; - use dotenv::dotenv; - use serial_test::serial; - use sqllogictest::{AsyncDB, DBOutput, DefaultColumnType}; use std::{ path::Path, sync::Arc, time::{Duration, Instant}, }; + + use anyhow::Result; + use async_trait::async_trait; + use dotenv::dotenv; + use serial_test::serial; + use sqllogictest::{AsyncDB, DBOutput, DefaultColumnType}; use timefusion::database::Database; use tokio::{sync::Notify, time::sleep}; use tokio_postgres::{NoTls, Row};