From a23846318e7a2295eb88529fab63840c2a25a398 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 2 Oct 2025 15:27:34 -0400 Subject: [PATCH 01/56] Basic containers.rs test module. Basic KafkaContex. --- Cargo.lock | 1891 ++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 2 + tests/containers.rs | 23 + 3 files changed, 1818 insertions(+), 98 deletions(-) create mode 100644 tests/containers.rs diff --git a/Cargo.lock b/Cargo.lock index 2b61633c2..f9d1aca74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + [[package]] name = "async-attributes" version = "1.1.2" @@ -176,7 +182,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix", + "rustix 0.38.37", "slab", "tracing", "windows-sys 0.59.0", @@ -219,7 +225,7 @@ dependencies = [ "cfg-if", "event-listener 5.3.1", "futures-lite", - "rustix", + "rustix 0.38.37", "tracing", ] @@ -235,7 +241,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix", + "rustix 0.38.37", "signal-hook-registry", "slab", "windows-sys 0.59.0", @@ -268,12 +274,45 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "async-task" version = "4.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -286,6 +325,49 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backon" version = "1.5.0" @@ -307,7 +389,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -316,13 +398,19 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bindgen" version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags", + "bitflags 2.6.0", "cexpr", "clang-sys", "itertools", @@ -334,6 +422,12 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.6.0" @@ -353,6 +447,82 @@ dependencies = [ "piper", ] +[[package]] +name = "bollard" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899ca34eb6924d6ec2a77c6f7f5c7339e60fd68235eaf91edd5a15f12958bb06" +dependencies = [ + "async-stream", + "base64 0.22.1", + "bitflags 2.6.0", + "bollard-buildkit-proto", + "bollard-stubs", + "bytes", + "chrono", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "num", + "pin-project-lite", + "rand", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-buildkit-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b3e79f8bd0f25f32660e3402afca46fd91bebaf135af017326d905651f8107" +dependencies = [ + "prost", + "prost-types", + "tonic", + "ureq", +] + +[[package]] +name = "bollard-stubs" +version = "1.48.3-rc.28.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ea257e555d16a2c01e5593f40b73865cdf12efbceda33c6d14a2d8d1490368" +dependencies = [ + "base64 0.22.1", + "bollard-buildkit-proto", + "bytes", + "chrono", + "prost", + "serde", + "serde_json", + "serde_repr", + "serde_with", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -365,12 +535,19 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "cc" -version = "1.1.21" +version = "1.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +checksum = "e1354349954c6fc9cb0deab020f27f783cf0b604e8bb754dc4658ecf0d29c35f" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -401,8 +578,9 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.1.1", ] [[package]] @@ -467,6 +645,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -512,6 +700,73 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.101", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "deranged" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "docker_credential" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d89dfcba45b4afad7450a99b39e751590463e45c04728cf555d36bb66940de8" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + [[package]] name = "duct" version = "0.13.7" @@ -524,6 +779,12 @@ dependencies = [ "shared_child", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.15.0" @@ -561,12 +822,23 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "etcetera" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c7b13d0780cb82722fd59f6f57f925e143427e4a75313a6c77243bf5326ae6" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.59.0", ] [[package]] @@ -602,6 +874,24 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "filetime" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.60.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" + [[package]] name = "flate2" version = "1.0.33" @@ -612,6 +902,21 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures" version = "0.3.31" @@ -714,6 +1019,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.3.3" @@ -723,7 +1039,7 @@ dependencies = [ "cfg-if", "libc", "r-efi", - "wasi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -750,6 +1066,31 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.5.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -762,7 +1103,7 @@ version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" dependencies = [ - "base64", + "base64 0.21.7", "byteorder", "crossbeam-channel", "flate2", @@ -777,94 +1118,383 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" [[package]] -name = "iana-time-zone" -version = "0.1.61" +name = "hex" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" +name = "home" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "cc", + "windows-sys 0.59.0", ] [[package]] -name = "indexmap" -version = "2.5.0" +name = "http" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ - "equivalent", - "hashbrown", + "bytes", + "fnv", + "itoa", ] [[package]] -name = "is_terminal_polyfill" -version = "1.70.1" +name = "http-body" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] [[package]] -name = "itertools" -version = "0.13.0" +name = "http-body-util" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ - "either", + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", ] [[package]] -name = "itoa" -version = "1.0.11" +name = "httparse" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] -name = "jiff" -version = "0.2.14" +name = "httpdate" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde", -] +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] -name = "jiff-static" -version = "0.2.14" +name = "hyper" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.101", + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", ] [[package]] -name = "jobserver" -version = "0.1.32" +name = "hyper-named-pipe" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ - "libc", + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", ] [[package]] -name = "js-sys" +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2 0.6.0", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", + "serde", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jiff" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" @@ -903,7 +1533,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a793df0d7afeac54f95b471d3af7f0d4fb975699f972341a4b76988d49cdf0c" dependencies = [ "cfg-if", - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "libredox" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +dependencies = [ + "bitflags 2.6.0", + "libc", + "redox_syscall 0.5.17", ] [[package]] @@ -924,6 +1565,18 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + [[package]] name = "log" version = "0.4.27" @@ -949,12 +1602,24 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -970,6 +1635,17 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + [[package]] name = "nom" version = "7.1.3" @@ -980,6 +1656,76 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1025,6 +1771,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + [[package]] name = "openssl-src" version = "300.3.2+3.3.2" @@ -1064,10 +1816,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] -name = "pin-project-lite" -version = "0.2.14" +name = "parse-display" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.101", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -1102,7 +1905,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix", + "rustix 0.38.37", "tracing", "windows-sys 0.59.0", ] @@ -1122,6 +1925,21 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1150,6 +1968,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.40" @@ -1191,13 +2041,14 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom", + "getrandom 0.3.3", ] [[package]] name = "rdkafka" version = "0.38.0" dependencies = [ + "anyhow", "async-std", "backon", "chrono", @@ -1219,6 +2070,7 @@ dependencies = [ "serde_json", "slab", "smol", + "testcontainers-modules", "tokio", "tracing", ] @@ -1239,6 +2091,44 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "regex" version = "1.11.1" @@ -1268,6 +2158,20 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1286,13 +2190,82 @@ version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ - "bitflags", + "bitflags 2.6.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.14", "windows-sys 0.52.0", ] +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys 0.11.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10b3f4191e8a80e6b43eebabfac91e5dcecebb27a71f04e820c47ec41d314bf" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.18" @@ -1312,20 +2285,86 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.1", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -1344,6 +2383,61 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.5.0", + "schemars 0.9.0", + "schemars 1.0.4", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "shared_child" version = "1.0.1" @@ -1378,6 +2472,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + [[package]] name = "smol" version = "2.0.2" @@ -1395,32 +2495,204 @@ dependencies = [ "futures-lite", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.101", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "testcontainers" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b92bce247dc9260a19808321e11b51ea6a0293d02b48ab1c6578960610cfa2a7" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util", + "ulid", + "url", +] + +[[package]] +name = "testcontainers-modules" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1966329d5bb3f89d33602d2db2da971fb839f9297dad16527abf4564e2ae0a6d" +dependencies = [ + "testcontainers", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", ] [[package]] -name = "syn" -version = "2.0.101" +name = "tinystr" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "displaydoc", + "zerovec", ] [[package]] @@ -1430,8 +2702,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" dependencies = [ "backtrace", + "bytes", + "libc", + "mio", "pin-project-lite", + "socket2 0.5.10", "tokio-macros", + "windows-sys 0.52.0", ] [[package]] @@ -1445,6 +2722,55 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "tokio-util" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml_datetime" version = "0.6.8" @@ -1457,11 +2783,71 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap", + "indexmap 2.5.0", "toml_datetime", "winnow", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.5.0", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.41" @@ -1493,12 +2879,67 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand", + "web-time", +] + [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "url" +version = "2.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1517,6 +2958,21 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasi" version = "0.14.2+wasi-0.2.4" @@ -1603,13 +3059,63 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.2", +] + +[[package]] +name = "webpki-roots" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1618,13 +3124,19 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1633,7 +3145,25 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.4", +] + +[[package]] +name = "windows-sys" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f109e41dd4a3c848907eb83d5a42ea98b3769495597450cf6d153507b166f0f" +dependencies = [ + "windows-link 0.2.0", ] [[package]] @@ -1642,14 +3172,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42b7b7f66d2a06854650af09cfdf8713e427a439c97ad65a6375318033ac4b" +dependencies = [ + "windows-link 0.2.0", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -1658,48 +3205,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.5.40" @@ -1715,7 +3310,47 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags", + "bitflags 2.6.0", +] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.2", +] + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", + "synstructure", ] [[package]] @@ -1738,6 +3373,66 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "zstd-sys" version = "2.0.15+zstd.1.5.7" diff --git a/Cargo.toml b/Cargo.toml index e556fd18c..2eadee201 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ rand = "0.9.1" regex = "1.11.1" smol = "2.0.2" tokio = { version = "1.18", features = ["macros", "rt-multi-thread", "time"] } +testcontainers-modules = { version = "0.13.0", features = ["kafka"] } +anyhow = { version = "1.0.100" } # These features are re-exports of the features that the rdkafka-sys crate # provides. See the rdkafka-sys documentation for details. diff --git a/tests/containers.rs b/tests/containers.rs new file mode 100644 index 000000000..08d88e762 --- /dev/null +++ b/tests/containers.rs @@ -0,0 +1,23 @@ +use anyhow::Context; +use testcontainers_modules::kafka::apache::Kafka; +use testcontainers_modules::testcontainers::runners::AsyncRunner; +use testcontainers_modules::testcontainers::ContainerAsync; + +pub struct KafkaContext { + kafka_node: ContainerAsync, +} + +impl KafkaContext { + pub async fn new() -> anyhow::Result { + let kafka_node = Kafka::default() + .start() + .await + .context("Failed to start Kafka")?; + + Ok(Self { kafka_node }) + } + + pub async fn bootstrap_servers(&self) -> String { + unimplemented!("not yet implemented") + } +} From 2351d8fa435e970d915f6c2089c58a0209564182 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 2 Oct 2025 15:37:46 -0400 Subject: [PATCH 02/56] Get bootstrap servers. --- tests/containers.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/containers.rs b/tests/containers.rs index 08d88e762..7e7061c4c 100644 --- a/tests/containers.rs +++ b/tests/containers.rs @@ -1,5 +1,6 @@ use anyhow::Context; use testcontainers_modules::kafka::apache::Kafka; +use testcontainers_modules::testcontainers::core::ContainerPort; use testcontainers_modules::testcontainers::runners::AsyncRunner; use testcontainers_modules::testcontainers::ContainerAsync; @@ -17,7 +18,16 @@ impl KafkaContext { Ok(Self { kafka_node }) } - pub async fn bootstrap_servers(&self) -> String { - unimplemented!("not yet implemented") + pub async fn bootstrap_servers(&self) -> anyhow::Result { + let kafka_host = self + .kafka_node + .get_host() + .await + .context("Failed to get Kafka host")?; + let kafka_port = self + .kafka_node + .get_host_port_ipv4(ContainerPort::Tcp(9092)) + .await?; + Ok(format!("{}:{}", kafka_host.to_string(), kafka_port)) } } From ea554b7369d350f06c10467b03f8a0b87c341871 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 2 Oct 2025 15:47:17 -0400 Subject: [PATCH 03/56] Basic test for KafkaContext based on TestContainers. --- tests/containers.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/containers.rs b/tests/containers.rs index 7e7061c4c..389c8573b 100644 --- a/tests/containers.rs +++ b/tests/containers.rs @@ -1,4 +1,5 @@ use anyhow::Context; +use std::fmt::Debug; use testcontainers_modules::kafka::apache::Kafka; use testcontainers_modules::testcontainers::core::ContainerPort; use testcontainers_modules::testcontainers::runners::AsyncRunner; @@ -8,6 +9,12 @@ pub struct KafkaContext { kafka_node: ContainerAsync, } +impl Debug for KafkaContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("KafkaContext").finish() + } +} + impl KafkaContext { pub async fn new() -> anyhow::Result { let kafka_node = Kafka::default() @@ -31,3 +38,24 @@ impl KafkaContext { Ok(format!("{}:{}", kafka_host.to_string(), kafka_port)) } } + +#[tokio::test] +pub async fn test_kafka_context_works() { + let kafka_context_result = KafkaContext::new().await; + let Ok(kafka_context) = kafka_context_result else { + panic!( + "Failed to get Kafka context: {}", + kafka_context_result.unwrap_err() + ); + }; + + let bootstrap_servers_result = kafka_context.bootstrap_servers().await; + let Ok(bootstrap_servers) = bootstrap_servers_result else { + panic!( + "Failed to get bootstrap servers: {}", + bootstrap_servers_result.unwrap_err() + ); + }; + + assert_ne!(bootstrap_servers.len(), 0, "Bootstrap servers empty"); +} From b4a0ac5e37305b6eb6d72477f2d3bff350c5d87a Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 2 Oct 2025 16:28:57 -0400 Subject: [PATCH 04/56] Some test functionality organization. --- tests/producer.rs | 17 +++++++++++++++++ tests/test_admin.rs | 4 +++- tests/test_high_consumers.rs | 3 ++- tests/test_high_producers.rs | 2 +- tests/test_low_consumers.rs | 1 + tests/test_low_producers.rs | 6 ++++-- tests/test_metadata.rs | 1 + tests/test_transactions.rs | 6 ++++-- tests/{ => utils}/containers.rs | 0 tests/utils/logging.rs | 9 +++++++++ tests/{utils.rs => utils/mod.rs} | 31 +++++-------------------------- tests/utils/rand.rs | 16 ++++++++++++++++ 12 files changed, 63 insertions(+), 33 deletions(-) create mode 100644 tests/producer.rs rename tests/{ => utils}/containers.rs (100%) create mode 100644 tests/utils/logging.rs rename tests/{utils.rs => utils/mod.rs} (89%) create mode 100644 tests/utils/rand.rs diff --git a/tests/producer.rs b/tests/producer.rs new file mode 100644 index 000000000..4469caca9 --- /dev/null +++ b/tests/producer.rs @@ -0,0 +1,17 @@ +use crate::utils::containers::KafkaContext; +use crate::utils::logging::init_test_logger; + +mod utils; + +#[tokio::test] +pub async fn test_basic_produce() { + init_test_logger(); + + let kafka_context_result = KafkaContext::new().await; + let Ok(_kafka_context) = kafka_context_result else { + panic!( + "could not create kafka context: {}", + kafka_context_result.unwrap_err() + ); + }; +} diff --git a/tests/test_admin.rs b/tests/test_admin.rs index 87f258a9e..a44921f5b 100644 --- a/tests/test_admin.rs +++ b/tests/test_admin.rs @@ -15,6 +15,8 @@ use rdkafka::metadata::Metadata; use rdkafka::producer::{FutureProducer, FutureRecord, Producer}; use rdkafka::{ClientConfig, Offset, TopicPartitionList}; +use crate::utils::logging::init_test_logger; +use crate::utils::rand::{rand_test_group, rand_test_topic}; use crate::utils::*; mod utils; @@ -26,7 +28,7 @@ fn create_config() -> ClientConfig { } fn create_admin_client() -> AdminClient { - configure_logging_for_tests(); + init_test_logger(); create_config() .create() .expect("admin client creation failed") diff --git a/tests/test_high_consumers.rs b/tests/test_high_consumers.rs index b22dc0b2b..88890a447 100644 --- a/tests/test_high_consumers.rs +++ b/tests/test_high_consumers.rs @@ -17,6 +17,7 @@ use rdkafka::util::current_time_millis; use rdkafka::{Message, Timestamp}; use rdkafka_sys::types::RDKafkaConfRes; +use crate::utils::rand::*; use crate::utils::*; mod utils; @@ -46,7 +47,7 @@ where #[tokio::test] async fn test_invalid_max_poll_interval() { let res: Result = consumer_config( - &rand_test_group(), + &crate::utils::rand::rand_test_group(), Some(hashmap! { "max.poll.interval.ms" => "-1" }), ) .create(); diff --git a/tests/test_high_producers.rs b/tests/test_high_producers.rs index 9a71c9981..d12c7d1ee 100644 --- a/tests/test_high_producers.rs +++ b/tests/test_high_producers.rs @@ -13,7 +13,7 @@ use rdkafka::producer::{FutureProducer, FutureRecord, Producer}; use rdkafka::util::Timeout; use rdkafka::Timestamp; -use crate::utils::*; +use crate::utils::rand::*; mod utils; diff --git a/tests/test_low_consumers.rs b/tests/test_low_consumers.rs index c97802243..307e2d824 100644 --- a/tests/test_low_consumers.rs +++ b/tests/test_low_consumers.rs @@ -13,6 +13,7 @@ use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::{current_time_millis, Timeout}; use rdkafka::{ClientConfig, Message, Timestamp}; +use crate::utils::rand::*; use crate::utils::*; mod utils; diff --git a/tests/test_low_producers.rs b/tests/test_low_producers.rs index ac12b8e04..c46c99681 100644 --- a/tests/test_low_producers.rs +++ b/tests/test_low_producers.rs @@ -21,6 +21,8 @@ use rdkafka::types::RDKafkaRespErr; use rdkafka::util::current_time_millis; use rdkafka::{ClientContext, Statistics}; +use crate::utils::logging::init_test_logger; +use crate::utils::rand::*; use crate::utils::*; mod utils; @@ -162,7 +164,7 @@ fn base_producer_with_context>( context: C, config_overrides: HashMap<&str, &str>, ) -> BaseProducer { - configure_logging_for_tests(); + init_test_logger(); default_config(config_overrides) .create_with_context::>(context) .unwrap() @@ -183,7 +185,7 @@ where Part: Partitioner + Send + Sync + 'static, C: ProducerContext, { - configure_logging_for_tests(); + init_test_logger(); default_config(config_overrides) .create_with_context::>(context) .unwrap() diff --git a/tests/test_metadata.rs b/tests/test_metadata.rs index eab0731b5..29449f2cb 100644 --- a/tests/test_metadata.rs +++ b/tests/test_metadata.rs @@ -9,6 +9,7 @@ use rdkafka::topic_partition_list::TopicPartitionList; use rdkafka_sys::types::RDKafkaConfRes; +use crate::utils::rand::*; use crate::utils::*; mod utils; diff --git a/tests/test_transactions.rs b/tests/test_transactions.rs index 014c4e000..90857f585 100644 --- a/tests/test_transactions.rs +++ b/tests/test_transactions.rs @@ -15,6 +15,8 @@ use rdkafka::producer::{BaseProducer, BaseRecord, Producer}; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::Timeout; +use crate::utils::logging::init_test_logger; +use crate::utils::rand::*; use utils::*; mod utils; @@ -22,12 +24,12 @@ mod utils; fn create_consumer( config_overrides: Option>, ) -> Result { - configure_logging_for_tests(); + init_test_logger(); consumer_config(&rand_test_group(), config_overrides).create() } fn create_producer() -> Result { - configure_logging_for_tests(); + init_test_logger(); let mut config = ClientConfig::new(); config .set("bootstrap.servers", get_bootstrap_server()) diff --git a/tests/containers.rs b/tests/utils/containers.rs similarity index 100% rename from tests/containers.rs rename to tests/utils/containers.rs diff --git a/tests/utils/logging.rs b/tests/utils/logging.rs new file mode 100644 index 000000000..401db65a5 --- /dev/null +++ b/tests/utils/logging.rs @@ -0,0 +1,9 @@ +use std::sync::Once; + +static INIT: Once = Once::new(); + +pub fn init_test_logger() { + INIT.call_once(|| { + env_logger::try_init().expect("Failed to initialize env_logger"); + }); +} diff --git a/tests/utils.rs b/tests/utils/mod.rs similarity index 89% rename from tests/utils.rs rename to tests/utils/mod.rs index 10ab34cf5..be75d1f3d 100644 --- a/tests/utils.rs +++ b/tests/utils/mod.rs @@ -1,11 +1,13 @@ #![allow(dead_code)] +pub mod containers; +pub mod logging; +pub mod rand; + use std::collections::HashMap; use std::env::{self, VarError}; -use std::sync::Once; use std::time::Duration; -use rand::distr::{Alphanumeric, SampleString}; use regex::Regex; use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication}; @@ -18,21 +20,6 @@ use rdkafka::producer::{FutureProducer, FutureRecord}; use rdkafka::statistics::Statistics; use rdkafka::TopicPartitionList; -pub fn rand_test_topic(test_name: &str) -> String { - let id = Alphanumeric.sample_string(&mut rand::rng(), 10); - format!("__{}_{}", test_name, id) -} - -pub fn rand_test_group() -> String { - let id = Alphanumeric.sample_string(&mut rand::rng(), 10); - format!("__test_{}", id) -} - -pub fn rand_test_transactional_id() -> String { - let id = Alphanumeric.sample_string(&mut rand::rng(), 10); - format!("__test_{}", id) -} - pub fn get_bootstrap_server() -> String { env::var("KAFKA_HOST").unwrap_or_else(|_| "localhost:9092".to_owned()) } @@ -195,21 +182,13 @@ pub fn consumer_config( config } -static INIT: Once = Once::new(); - -pub fn configure_logging_for_tests() { - INIT.call_once(|| { - env_logger::try_init().expect("Failed to initialize env_logger"); - }); -} - #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_populate_topic() { - let topic_name = rand_test_topic("test_populate_topic"); + let topic_name = rand::rand_test_topic("test_populate_topic"); let message_map = populate_topic(&topic_name, 100, &value_fn, &key_fn, Some(0), None).await; let total_messages = message_map diff --git a/tests/utils/rand.rs b/tests/utils/rand.rs new file mode 100644 index 000000000..495d25ba0 --- /dev/null +++ b/tests/utils/rand.rs @@ -0,0 +1,16 @@ +use rand::distr::{Alphanumeric, SampleString}; + +pub fn rand_test_topic(test_name: &str) -> String { + let id = Alphanumeric.sample_string(&mut rand::rng(), 10); + format!("__{}_{}", test_name, id) +} + +pub fn rand_test_group() -> String { + let id = Alphanumeric.sample_string(&mut rand::rng(), 10); + format!("__test_{}", id) +} + +pub fn rand_test_transactional_id() -> String { + let id = Alphanumeric.sample_string(&mut rand::rng(), 10); + format!("__test_{}", id) +} From bef91368829032a67d04168c0ae12d54f10d3f0f Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 2 Oct 2025 17:17:03 -0400 Subject: [PATCH 05/56] Basic produce. --- src/producer/base_producer.rs | 14 ++++++++++++++ tests/producer.rs | 30 ++++++++++++++++++++++++++++++ tests/utils/mod.rs | 1 + tests/utils/producer.rs | 11 +++++++++++ 4 files changed, 56 insertions(+) create mode 100644 tests/utils/producer.rs diff --git a/src/producer/base_producer.rs b/src/producer/base_producer.rs index 0841bafba..7f6d82478 100644 --- a/src/producer/base_producer.rs +++ b/src/producer/base_producer.rs @@ -42,6 +42,7 @@ //! should wait and try again. use std::ffi::{CStr, CString}; +use std::fmt; use std::marker::PhantomData; use std::mem; use std::os::raw::c_void; @@ -340,6 +341,19 @@ where _partitioner: PhantomData, } +impl fmt::Debug for BaseProducer +where + Part: Partitioner, + C: ProducerContext, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BaseProducer") + .field("native_ptr", &self.native_ptr()) + .field("queue", &self.queue) + .finish() + } +} + impl BaseProducer where Part: Partitioner, diff --git a/tests/producer.rs b/tests/producer.rs index 4469caca9..49d9471e6 100644 --- a/tests/producer.rs +++ b/tests/producer.rs @@ -1,5 +1,9 @@ +use rdkafka::ClientConfig; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; +use crate::utils::producer::create_base_producer; +use crate::utils::rand::rand_test_topic; +use rdkafka::producer::BaseRecord; mod utils; @@ -14,4 +18,30 @@ pub async fn test_basic_produce() { kafka_context_result.unwrap_err() ); }; + + let bootstrap_servers_result = _kafka_context.bootstrap_servers().await; + let Ok(bootstrap_servers) = bootstrap_servers_result else { + panic!("could not create bootstrap servers: {}", bootstrap_servers_result.unwrap_err()); + }; + let mut client_config = ClientConfig::default(); + client_config.set("bootstrap.servers", bootstrap_servers); + + let base_producer_result = create_base_producer(&client_config); + let Ok(base_producer) = base_producer_result else { + panic!( + "could not create based_producer: {}", + base_producer_result.unwrap_err() + ); + }; + + let test_topic = rand_test_topic("testing-topic"); + let record = BaseRecord::to(&test_topic) // destination topic + .key(&[1, 2, 3, 4]) // message key + .payload("content") // message payload + .partition(5); + + let send_result = base_producer.send(record); + if send_result.is_err() { + panic!("could not produce record: {:?}", send_result.unwrap_err()); + } } diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index be75d1f3d..4a07a88a2 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -2,6 +2,7 @@ pub mod containers; pub mod logging; +pub mod producer; pub mod rand; use std::collections::HashMap; diff --git a/tests/utils/producer.rs b/tests/utils/producer.rs new file mode 100644 index 000000000..0398eb21e --- /dev/null +++ b/tests/utils/producer.rs @@ -0,0 +1,11 @@ +use rdkafka::ClientConfig; +use rdkafka::config::FromClientConfig; +use rdkafka::producer::BaseProducer; + +pub fn create_base_producer(config: &ClientConfig) -> anyhow::Result { + let base_producer_result = BaseProducer::from_config(config); + let Ok(base_producer) = base_producer_result else { + anyhow::bail!("error creating base producer: {}", base_producer_result.unwrap_err()) + }; + Ok(base_producer) +} From 1cf7042329e9f4a1c2382ed9a44c9f96743d8117 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 9 Oct 2025 22:16:19 -0400 Subject: [PATCH 06/56] Various updates. KafkaContext is now shared. --- src/consumer/base_consumer.rs | 15 +++++++ tests/producer.rs | 81 ++++++++++++++++++++++++++++++----- tests/utils/consumer.rs | 17 ++++++++ tests/utils/containers.rs | 24 ++++++++--- tests/utils/mod.rs | 1 + tests/utils/producer.rs | 8 +++- 6 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 tests/utils/consumer.rs diff --git a/src/consumer/base_consumer.rs b/src/consumer/base_consumer.rs index a25e36611..f69ccffc0 100644 --- a/src/consumer/base_consumer.rs +++ b/src/consumer/base_consumer.rs @@ -1,6 +1,7 @@ //! Low-level consumers. use std::ffi::{CStr, CString}; +use std::fmt; use std::mem::ManuallyDrop; use std::os::raw::c_void; use std::ptr; @@ -41,6 +42,20 @@ where nonempty_callback: Option>>, } +impl fmt::Debug for BaseConsumer +where + C: ConsumerContext, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BaseConsumer") + .field("native_ptr", &self.client.native_ptr()) + .field("queue", &self.queue) + .field("group_id", &self.group_id) + .field("has_nonempty_callback", &self.nonempty_callback.is_some()) + .finish() + } +} + impl FromClientConfig for BaseConsumer { fn from_config(config: &ClientConfig) -> KafkaResult { BaseConsumer::from_config_and_context(config, DefaultConsumerContext) diff --git a/tests/producer.rs b/tests/producer.rs index 49d9471e6..ed096ff26 100644 --- a/tests/producer.rs +++ b/tests/producer.rs @@ -1,9 +1,14 @@ -use rdkafka::ClientConfig; +use crate::utils::consumer; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; use crate::utils::producer::create_base_producer; -use crate::utils::rand::rand_test_topic; -use rdkafka::producer::BaseRecord; +use crate::utils::rand::{rand_test_group, rand_test_topic}; +use rdkafka::consumer::{BaseConsumer, Consumer}; +use rdkafka::error::KafkaError; +use rdkafka::producer::{BaseRecord, Producer}; +use rdkafka::util::Timeout; +use rdkafka::{ClientConfig, Message}; +use std::time::Duration; mod utils; @@ -21,12 +26,19 @@ pub async fn test_basic_produce() { let bootstrap_servers_result = _kafka_context.bootstrap_servers().await; let Ok(bootstrap_servers) = bootstrap_servers_result else { - panic!("could not create bootstrap servers: {}", bootstrap_servers_result.unwrap_err()); + panic!( + "could not create bootstrap servers: {}", + bootstrap_servers_result.unwrap_err() + ); }; - let mut client_config = ClientConfig::default(); - client_config.set("bootstrap.servers", bootstrap_servers); + let test_topic = rand_test_topic("testing-topic"); - let base_producer_result = create_base_producer(&client_config); + create_consumer(bootstrap_servers, test_topic); + + let mut producer_client_config = ClientConfig::default(); + producer_client_config.set("bootstrap.servers", &bootstrap_servers); + + let base_producer_result = create_base_producer(&producer_client_config); let Ok(base_producer) = base_producer_result else { panic!( "could not create based_producer: {}", @@ -34,14 +46,63 @@ pub async fn test_basic_produce() { ); }; - let test_topic = rand_test_topic("testing-topic"); let record = BaseRecord::to(&test_topic) // destination topic .key(&[1, 2, 3, 4]) // message key - .payload("content") // message payload - .partition(5); + .payload("content"); // message payload let send_result = base_producer.send(record); if send_result.is_err() { panic!("could not produce record: {:?}", send_result.unwrap_err()); } + + let flush_result = base_producer.flush(Timeout::After(Duration::from_secs(10))); + if let Err(flush_error) = flush_result { + panic!("timed out waiting for producer flush: {}", flush_error); + } + + let Some(next_message_result) = base_consumer.poll(Duration::from_secs(2)) else { + panic!("there is no next message on the topic: {}", test_topic); + }; + let Ok(borrowed_next_message) = next_message_result else { + panic!( + "could not get next message from based_consumer: {}", + next_message_result.unwrap_err() + ); + }; + let owned_next_message = borrowed_next_message.detach(); + let Some(message_payload) = owned_next_message.payload() else { + panic!("message payload is empty"); + }; + let message_string_result = String::from_utf8(message_payload.to_vec()); + let Ok(message_string) = message_string_result else { + panic!("message payload is not valid UTF-8"); + }; + + assert!(message_string.contains("content")); +} + +async fn create_consumer( + bootstrap_servers: String, + test_topic: String, +) -> anyhow::Result { + let mut consumer_client_config = ClientConfig::default(); + consumer_client_config.set("group.id", rand_test_group()); + consumer_client_config.set("client.id", "rdkafka_integration_test_client"); + consumer_client_config.set("bootstrap.servers", &bootstrap_servers); + consumer_client_config.set("enable.partition.eof", "false"); + consumer_client_config.set("session.timeout.ms", "6000"); + consumer_client_config.set("enable.auto.commit", "false"); + consumer_client_config.set("debug", "all"); + consumer_client_config.set("auto.offset.reset", "earliest"); + + let base_consumer_result = + consumer::create_subscribed_base(consumer_client_config, &[&test_topic]).await; + let Ok(base_consumer) = base_consumer_result else { + panic!( + "could not create base consumer: {}", + base_consumer_result.unwrap_err() + ) + }; + + unimplemented!("unimplemented"); } diff --git a/tests/utils/consumer.rs b/tests/utils/consumer.rs new file mode 100644 index 000000000..187728527 --- /dev/null +++ b/tests/utils/consumer.rs @@ -0,0 +1,17 @@ +use anyhow::Context; +use rdkafka::config::FromClientConfig; +use rdkafka::consumer::{BaseConsumer, Consumer}; +use rdkafka::ClientConfig; + +pub async fn create_subscribed_base( + client_config: ClientConfig, + topics: &[&str], +) -> anyhow::Result { + let base_consumer = + BaseConsumer::from_config(&client_config).context("Failed to create consumer")?; + base_consumer + .subscribe(topics) + .context("Failed to subscribe to topic")?; + + Ok(base_consumer) +} diff --git a/tests/utils/containers.rs b/tests/utils/containers.rs index 389c8573b..39811c4bb 100644 --- a/tests/utils/containers.rs +++ b/tests/utils/containers.rs @@ -1,9 +1,11 @@ use anyhow::Context; use std::fmt::Debug; +use std::sync::Arc; use testcontainers_modules::kafka::apache::Kafka; use testcontainers_modules::testcontainers::core::ContainerPort; use testcontainers_modules::testcontainers::runners::AsyncRunner; use testcontainers_modules::testcontainers::ContainerAsync; +use tokio::sync::OnceCell; pub struct KafkaContext { kafka_node: ContainerAsync, @@ -16,13 +18,21 @@ impl Debug for KafkaContext { } impl KafkaContext { - pub async fn new() -> anyhow::Result { - let kafka_node = Kafka::default() - .start() - .await - .context("Failed to start Kafka")?; + pub async fn shared() -> anyhow::Result> { + static INSTANCE: OnceCell> = OnceCell::const_new(); + + INSTANCE + .get_or_try_init(|| async { + let kafka_node = Kafka::default() + .start() + .await + .context("Failed to start Kafka")?; - Ok(Self { kafka_node }) + Ok::, anyhow::Error>(Arc::new(KafkaContext { kafka_node })) + }) + .await + .context("Failed to initialize Kafka shared instance") + .map(Arc::clone) } pub async fn bootstrap_servers(&self) -> anyhow::Result { @@ -41,7 +51,7 @@ impl KafkaContext { #[tokio::test] pub async fn test_kafka_context_works() { - let kafka_context_result = KafkaContext::new().await; + let kafka_context_result = KafkaContext::shared().await; let Ok(kafka_context) = kafka_context_result else { panic!( "Failed to get Kafka context: {}", diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index 4a07a88a2..8b12c5b4d 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -1,5 +1,6 @@ #![allow(dead_code)] +pub mod consumer; pub mod containers; pub mod logging; pub mod producer; diff --git a/tests/utils/producer.rs b/tests/utils/producer.rs index 0398eb21e..d553edcdd 100644 --- a/tests/utils/producer.rs +++ b/tests/utils/producer.rs @@ -1,11 +1,15 @@ -use rdkafka::ClientConfig; use rdkafka::config::FromClientConfig; use rdkafka::producer::BaseProducer; +use rdkafka::ClientConfig; pub fn create_base_producer(config: &ClientConfig) -> anyhow::Result { let base_producer_result = BaseProducer::from_config(config); let Ok(base_producer) = base_producer_result else { - anyhow::bail!("error creating base producer: {}", base_producer_result.unwrap_err()) + anyhow::bail!( + "error creating base producer: {}", + base_producer_result.unwrap_err() + ) }; + Ok(base_producer) } From 92711d484df764dbd652d181592b4b2265067bbb Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 10 Oct 2025 19:13:14 -0400 Subject: [PATCH 07/56] admin. test topic creation. --- tests/admin.rs | 48 ++++++++++++++++++++++++++++ tests/producer.rs | 64 +++++++++++++++++++++---------------- tests/test_low_consumers.rs | 1 + tests/utils/admin.rs | 13 ++++++++ tests/utils/containers.rs | 55 +++++++++++++++---------------- tests/utils/mod.rs | 1 + 6 files changed, 125 insertions(+), 57 deletions(-) create mode 100644 tests/admin.rs create mode 100644 tests/utils/admin.rs diff --git a/tests/admin.rs b/tests/admin.rs new file mode 100644 index 000000000..314e8d396 --- /dev/null +++ b/tests/admin.rs @@ -0,0 +1,48 @@ +use rdkafka::admin::{AdminOptions, NewTopic, TopicReplication}; +use crate::utils::containers::KafkaContext; +use crate::utils::logging::init_test_logger; +use crate::utils::rand::rand_test_topic; + +#[path = "utils/mod.rs"] +mod utils; + +#[tokio::test] +pub async fn test_topic_creation() { + init_test_logger(); + + // Get Kafka container context. + let kafka_context_result = KafkaContext::shared().await; + let Ok(kafka_context) = kafka_context_result else { + panic!( + "could not create kafka context: {}", + kafka_context_result.unwrap_err() + ); + }; + let test_topic_name = rand_test_topic("testing-topic"); + + let admin_client_result = utils::admin::create_admin(&kafka_context.bootstrap_servers).await; + let Ok(admin_client) = admin_client_result else { + panic!( + "could not create admin client: {}", + admin_client_result.unwrap_err() + ); + }; + + let new_topic = NewTopic::new(&test_topic_name, 1, TopicReplication::Fixed(1)); + let admin_opts = AdminOptions::new(); + let create_topics_result = admin_client + .create_topics(vec![&new_topic], &admin_opts) + .await; + let Ok(create_topics) = create_topics_result else { + panic!( + "could not create new topics: {}", + create_topics_result.unwrap_err() + ); + }; + + for topic_result in create_topics { + if topic_result.is_err() { + panic!("failed to create topic: {:?}", topic_result.unwrap_err()); + }; + } +} diff --git a/tests/producer.rs b/tests/producer.rs index ed096ff26..aebf2c4bb 100644 --- a/tests/producer.rs +++ b/tests/producer.rs @@ -3,9 +3,10 @@ use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; use crate::utils::producer::create_base_producer; use crate::utils::rand::{rand_test_group, rand_test_topic}; -use rdkafka::consumer::{BaseConsumer, Consumer}; -use rdkafka::error::KafkaError; -use rdkafka::producer::{BaseRecord, Producer}; +use anyhow::Context; +use rdkafka::config::FromClientConfig; +use rdkafka::consumer::BaseConsumer; +use rdkafka::producer::{BaseProducer, BaseRecord, Producer}; use rdkafka::util::Timeout; use rdkafka::{ClientConfig, Message}; use std::time::Duration; @@ -16,37 +17,32 @@ mod utils; pub async fn test_basic_produce() { init_test_logger(); - let kafka_context_result = KafkaContext::new().await; - let Ok(_kafka_context) = kafka_context_result else { + // Get Kafka container context. + let kafka_context_result = KafkaContext::shared().await; + let Ok(kafka_context) = kafka_context_result else { panic!( "could not create kafka context: {}", kafka_context_result.unwrap_err() ); }; + let test_topic_name = rand_test_topic("testing-topic"); - let bootstrap_servers_result = _kafka_context.bootstrap_servers().await; - let Ok(bootstrap_servers) = bootstrap_servers_result else { + let consumer_result = create_consumer(&kafka_context.bootstrap_servers, &test_topic_name).await; + let Ok(consumer) = consumer_result else { panic!( - "could not create bootstrap servers: {}", - bootstrap_servers_result.unwrap_err() + "could not create consumer: {}", + consumer_result.unwrap_err() ); }; - let test_topic = rand_test_topic("testing-topic"); - - create_consumer(bootstrap_servers, test_topic); - - let mut producer_client_config = ClientConfig::default(); - producer_client_config.set("bootstrap.servers", &bootstrap_servers); - - let base_producer_result = create_base_producer(&producer_client_config); - let Ok(base_producer) = base_producer_result else { + let create_producer_result = create_producer(&kafka_context.bootstrap_servers).await; + let Ok(base_producer) = create_producer_result else { panic!( - "could not create based_producer: {}", - base_producer_result.unwrap_err() + "could not create base producer: {}", + create_producer_result.unwrap_err() ); }; - let record = BaseRecord::to(&test_topic) // destination topic + let record = BaseRecord::to(&test_topic_name) // destination topic .key(&[1, 2, 3, 4]) // message key .payload("content"); // message payload @@ -54,14 +50,13 @@ pub async fn test_basic_produce() { if send_result.is_err() { panic!("could not produce record: {:?}", send_result.unwrap_err()); } - let flush_result = base_producer.flush(Timeout::After(Duration::from_secs(10))); if let Err(flush_error) = flush_result { panic!("timed out waiting for producer flush: {}", flush_error); } - let Some(next_message_result) = base_consumer.poll(Duration::from_secs(2)) else { - panic!("there is no next message on the topic: {}", test_topic); + let Some(next_message_result) = consumer.poll(Duration::from_secs(2)) else { + panic!("there is no next message on the topic: {}", test_topic_name); }; let Ok(borrowed_next_message) = next_message_result else { panic!( @@ -82,13 +77,13 @@ pub async fn test_basic_produce() { } async fn create_consumer( - bootstrap_servers: String, - test_topic: String, + bootstrap_servers: &str, + test_topic: &str, ) -> anyhow::Result { let mut consumer_client_config = ClientConfig::default(); consumer_client_config.set("group.id", rand_test_group()); consumer_client_config.set("client.id", "rdkafka_integration_test_client"); - consumer_client_config.set("bootstrap.servers", &bootstrap_servers); + consumer_client_config.set("bootstrap.servers", bootstrap_servers); consumer_client_config.set("enable.partition.eof", "false"); consumer_client_config.set("session.timeout.ms", "6000"); consumer_client_config.set("enable.auto.commit", "false"); @@ -104,5 +99,18 @@ async fn create_consumer( ) }; - unimplemented!("unimplemented"); + Ok(base_consumer) +} + +async fn create_producer(bootstrap_servers: &str) -> anyhow::Result { + let mut producer_client_config = ClientConfig::default(); + producer_client_config.set("bootstrap.servers", bootstrap_servers); + let base_producer_result = create_base_producer(&producer_client_config); + let Ok(base_producer) = base_producer_result else { + panic!( + "could not create based_producer: {}", + base_producer_result.unwrap_err() + ); + }; + Ok(base_producer) } diff --git a/tests/test_low_consumers.rs b/tests/test_low_consumers.rs index 307e2d824..3158c3497 100644 --- a/tests/test_low_consumers.rs +++ b/tests/test_low_consumers.rs @@ -16,6 +16,7 @@ use rdkafka::{ClientConfig, Message, Timestamp}; use crate::utils::rand::*; use crate::utils::*; +mod admin; mod utils; fn create_base_consumer( diff --git a/tests/utils/admin.rs b/tests/utils/admin.rs new file mode 100644 index 000000000..ed5148b0c --- /dev/null +++ b/tests/utils/admin.rs @@ -0,0 +1,13 @@ +use anyhow::Context; +use rdkafka::admin::AdminClient; +use rdkafka::client::DefaultClientContext; +use rdkafka::config::FromClientConfig; +use rdkafka::ClientConfig; + +pub async fn create_admin( + bootstrap_servers: &str, +) -> anyhow::Result> { + let mut admin_client_config = ClientConfig::default(); + admin_client_config.set("bootstrap.servers", bootstrap_servers); + AdminClient::from_config(&admin_client_config).context("error creating admin client") +} diff --git a/tests/utils/containers.rs b/tests/utils/containers.rs index 39811c4bb..f79a195f8 100644 --- a/tests/utils/containers.rs +++ b/tests/utils/containers.rs @@ -9,6 +9,7 @@ use tokio::sync::OnceCell; pub struct KafkaContext { kafka_node: ContainerAsync, + pub bootstrap_servers: String, } impl Debug for KafkaContext { @@ -22,31 +23,31 @@ impl KafkaContext { static INSTANCE: OnceCell> = OnceCell::const_new(); INSTANCE - .get_or_try_init(|| async { - let kafka_node = Kafka::default() - .start() - .await - .context("Failed to start Kafka")?; - - Ok::, anyhow::Error>(Arc::new(KafkaContext { kafka_node })) - }) + .get_or_try_init(init) .await .context("Failed to initialize Kafka shared instance") .map(Arc::clone) } +} - pub async fn bootstrap_servers(&self) -> anyhow::Result { - let kafka_host = self - .kafka_node - .get_host() - .await - .context("Failed to get Kafka host")?; - let kafka_port = self - .kafka_node - .get_host_port_ipv4(ContainerPort::Tcp(9092)) - .await?; - Ok(format!("{}:{}", kafka_host.to_string(), kafka_port)) - } +async fn init() -> anyhow::Result> { + let kafka_node = Kafka::default() + .start() + .await + .context("Failed to start Kafka")?; + + let kafka_host = kafka_node + .get_host() + .await + .context("Failed to get Kafka host")?; + let kafka_port = kafka_node + .get_host_port_ipv4(ContainerPort::Tcp(9092)) + .await?; + + Ok::, anyhow::Error>(Arc::new(KafkaContext { + kafka_node, + bootstrap_servers: format!("{}:{}", kafka_host, kafka_port), + })) } #[tokio::test] @@ -59,13 +60,9 @@ pub async fn test_kafka_context_works() { ); }; - let bootstrap_servers_result = kafka_context.bootstrap_servers().await; - let Ok(bootstrap_servers) = bootstrap_servers_result else { - panic!( - "Failed to get bootstrap servers: {}", - bootstrap_servers_result.unwrap_err() - ); - }; - - assert_ne!(bootstrap_servers.len(), 0, "Bootstrap servers empty"); + assert_ne!( + kafka_context.bootstrap_servers.len(), + 0, + "Bootstrap servers empty" + ); } diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index 8b12c5b4d..3522ebc87 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -1,5 +1,6 @@ #![allow(dead_code)] +pub mod admin; pub mod consumer; pub mod containers; pub mod logging; From f16e83b2ba3547b0c6c3c75c8f2b9723c9615ed4 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 10 Oct 2025 20:38:09 -0400 Subject: [PATCH 08/56] basic producer test works. --- tests/admin.rs | 23 +++------ tests/producer.rs | 94 +++++++++++++------------------------ tests/test_low_consumers.rs | 1 - tests/utils/admin.rs | 27 +++++++++-- tests/utils/consumer.rs | 54 +++++++++++++++++++++ tests/utils/containers.rs | 18 +++++++ tests/utils/producer.rs | 28 ++++++++++- tests/utils/rand.rs | 2 +- 8 files changed, 163 insertions(+), 84 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index 314e8d396..e57e11442 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -1,4 +1,4 @@ -use rdkafka::admin::{AdminOptions, NewTopic, TopicReplication}; +use crate::utils::admin::create_topic; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; use crate::utils::rand::rand_test_topic; @@ -20,7 +20,8 @@ pub async fn test_topic_creation() { }; let test_topic_name = rand_test_topic("testing-topic"); - let admin_client_result = utils::admin::create_admin(&kafka_context.bootstrap_servers).await; + let admin_client_result = + utils::admin::create_admin_client(&kafka_context.bootstrap_servers).await; let Ok(admin_client) = admin_client_result else { panic!( "could not create admin client: {}", @@ -28,21 +29,11 @@ pub async fn test_topic_creation() { ); }; - let new_topic = NewTopic::new(&test_topic_name, 1, TopicReplication::Fixed(1)); - let admin_opts = AdminOptions::new(); - let create_topics_result = admin_client - .create_topics(vec![&new_topic], &admin_opts) - .await; - let Ok(create_topics) = create_topics_result else { + let create_topic_result = create_topic(&admin_client, &test_topic_name).await; + if create_topic_result.is_err() { panic!( - "could not create new topics: {}", - create_topics_result.unwrap_err() + "could not create topic: {}", + create_topic_result.unwrap_err() ); }; - - for topic_result in create_topics { - if topic_result.is_err() { - panic!("failed to create topic: {:?}", topic_result.unwrap_err()); - }; - } } diff --git a/tests/producer.rs b/tests/producer.rs index aebf2c4bb..7987d2ecc 100644 --- a/tests/producer.rs +++ b/tests/producer.rs @@ -1,23 +1,19 @@ -use crate::utils::consumer; +use crate::utils::admin::{create_admin_client, create_topic}; +use crate::utils::consumer::{create_consumer, poll_x_times_for_messages}; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; -use crate::utils::producer::create_base_producer; -use crate::utils::rand::{rand_test_group, rand_test_topic}; -use anyhow::Context; -use rdkafka::config::FromClientConfig; -use rdkafka::consumer::BaseConsumer; -use rdkafka::producer::{BaseProducer, BaseRecord, Producer}; -use rdkafka::util::Timeout; -use rdkafka::{ClientConfig, Message}; -use std::time::Duration; +use crate::utils::producer::{create_producer, poll_and_flush}; +use crate::utils::rand::rand_test_topic; +use rdkafka::producer::BaseRecord; +use rdkafka::Message; +#[path = "utils/mod.rs"] mod utils; #[tokio::test] pub async fn test_basic_produce() { init_test_logger(); - // Get Kafka container context. let kafka_context_result = KafkaContext::shared().await; let Ok(kafka_context) = kafka_context_result else { panic!( @@ -27,6 +23,21 @@ pub async fn test_basic_produce() { }; let test_topic_name = rand_test_topic("testing-topic"); + let admin_client_result = create_admin_client(&kafka_context.bootstrap_servers).await; + let Ok(admin_client) = admin_client_result else { + panic!( + "could not create admin client: {}", + admin_client_result.unwrap_err() + ); + }; + let create_topic_result = create_topic(&admin_client, &test_topic_name).await; + if create_topic_result.is_err() { + panic!( + "could not create topic: {}", + create_topic_result.unwrap_err() + ); + } + let consumer_result = create_consumer(&kafka_context.bootstrap_servers, &test_topic_name).await; let Ok(consumer) = consumer_result else { panic!( @@ -34,6 +45,7 @@ pub async fn test_basic_produce() { consumer_result.unwrap_err() ); }; + let create_producer_result = create_producer(&kafka_context.bootstrap_servers).await; let Ok(base_producer) = create_producer_result else { panic!( @@ -50,20 +62,19 @@ pub async fn test_basic_produce() { if send_result.is_err() { panic!("could not produce record: {:?}", send_result.unwrap_err()); } - let flush_result = base_producer.flush(Timeout::After(Duration::from_secs(10))); - if let Err(flush_error) = flush_result { - panic!("timed out waiting for producer flush: {}", flush_error); + if poll_and_flush(&base_producer).is_err() { + panic!("could not poll and flush base producer") } - let Some(next_message_result) = consumer.poll(Duration::from_secs(2)) else { - panic!("there is no next message on the topic: {}", test_topic_name); - }; - let Ok(borrowed_next_message) = next_message_result else { - panic!( - "could not get next message from based_consumer: {}", - next_message_result.unwrap_err() - ); + let messages_result = poll_x_times_for_messages(&consumer, 10).await; + let Ok(messages) = messages_result else { + panic!("could not get messages from consumer"); }; + if messages.len() != 1 { + panic!("expected exactly one message"); + } + let borrowed_next_message = messages.get(0).unwrap(); + let owned_next_message = borrowed_next_message.detach(); let Some(message_payload) = owned_next_message.payload() else { panic!("message payload is empty"); @@ -75,42 +86,3 @@ pub async fn test_basic_produce() { assert!(message_string.contains("content")); } - -async fn create_consumer( - bootstrap_servers: &str, - test_topic: &str, -) -> anyhow::Result { - let mut consumer_client_config = ClientConfig::default(); - consumer_client_config.set("group.id", rand_test_group()); - consumer_client_config.set("client.id", "rdkafka_integration_test_client"); - consumer_client_config.set("bootstrap.servers", bootstrap_servers); - consumer_client_config.set("enable.partition.eof", "false"); - consumer_client_config.set("session.timeout.ms", "6000"); - consumer_client_config.set("enable.auto.commit", "false"); - consumer_client_config.set("debug", "all"); - consumer_client_config.set("auto.offset.reset", "earliest"); - - let base_consumer_result = - consumer::create_subscribed_base(consumer_client_config, &[&test_topic]).await; - let Ok(base_consumer) = base_consumer_result else { - panic!( - "could not create base consumer: {}", - base_consumer_result.unwrap_err() - ) - }; - - Ok(base_consumer) -} - -async fn create_producer(bootstrap_servers: &str) -> anyhow::Result { - let mut producer_client_config = ClientConfig::default(); - producer_client_config.set("bootstrap.servers", bootstrap_servers); - let base_producer_result = create_base_producer(&producer_client_config); - let Ok(base_producer) = base_producer_result else { - panic!( - "could not create based_producer: {}", - base_producer_result.unwrap_err() - ); - }; - Ok(base_producer) -} diff --git a/tests/test_low_consumers.rs b/tests/test_low_consumers.rs index 3158c3497..307e2d824 100644 --- a/tests/test_low_consumers.rs +++ b/tests/test_low_consumers.rs @@ -16,7 +16,6 @@ use rdkafka::{ClientConfig, Message, Timestamp}; use crate::utils::rand::*; use crate::utils::*; -mod admin; mod utils; fn create_base_consumer( diff --git a/tests/utils/admin.rs b/tests/utils/admin.rs index ed5148b0c..70e7295cf 100644 --- a/tests/utils/admin.rs +++ b/tests/utils/admin.rs @@ -1,13 +1,34 @@ -use anyhow::Context; -use rdkafka::admin::AdminClient; +use anyhow::{bail, Context}; +use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication}; use rdkafka::client::DefaultClientContext; use rdkafka::config::FromClientConfig; use rdkafka::ClientConfig; -pub async fn create_admin( +pub async fn create_admin_client( bootstrap_servers: &str, ) -> anyhow::Result> { let mut admin_client_config = ClientConfig::default(); admin_client_config.set("bootstrap.servers", bootstrap_servers); AdminClient::from_config(&admin_client_config).context("error creating admin client") } + +pub async fn create_topic( + admin_client: &AdminClient, + topic_name: &'_ str, +) -> anyhow::Result { + let topic_results = admin_client + .create_topics(&new_topic_vec(&topic_name), &AdminOptions::default()) + .await + .context("error creating topics")?; + for topic_result in topic_results { + if topic_result.is_err() { + bail!("failed to create topic: {:?}", topic_result.unwrap_err()); + }; + } + Ok(topic_name.to_string()) +} + +pub fn new_topic_vec(topic_name: &'_ str) -> Vec> { + let new_topic = NewTopic::new(&topic_name, 1, TopicReplication::Fixed(1)); + vec![new_topic] +} diff --git a/tests/utils/consumer.rs b/tests/utils/consumer.rs index 187728527..267b1e212 100644 --- a/tests/utils/consumer.rs +++ b/tests/utils/consumer.rs @@ -1,7 +1,37 @@ +use crate::utils::consumer; +use crate::utils::rand::rand_test_group; use anyhow::Context; use rdkafka::config::FromClientConfig; use rdkafka::consumer::{BaseConsumer, Consumer}; +use rdkafka::message::BorrowedMessage; use rdkafka::ClientConfig; +use std::time::Duration; + +pub async fn create_consumer( + bootstrap_servers: &str, + test_topic: &str, +) -> anyhow::Result { + let mut consumer_client_config = ClientConfig::default(); + consumer_client_config.set("group.id", rand_test_group()); + consumer_client_config.set("client.id", "rdkafka_integration_test_client"); + consumer_client_config.set("bootstrap.servers", bootstrap_servers); + consumer_client_config.set("enable.partition.eof", "false"); + consumer_client_config.set("session.timeout.ms", "6000"); + consumer_client_config.set("enable.auto.commit", "false"); + consumer_client_config.set("debug", "all"); + consumer_client_config.set("auto.offset.reset", "earliest"); + + let base_consumer_result = + consumer::create_subscribed_base(consumer_client_config, &[&test_topic]).await; + let Ok(base_consumer) = base_consumer_result else { + panic!( + "could not create base consumer: {}", + base_consumer_result.unwrap_err() + ) + }; + + Ok(base_consumer) +} pub async fn create_subscribed_base( client_config: ClientConfig, @@ -15,3 +45,27 @@ pub async fn create_subscribed_base( Ok(base_consumer) } + +pub async fn poll_x_times_for_messages( + consumer: &BaseConsumer, + times_to_poll: i32, +) -> anyhow::Result>> { + let mut borrowed_messages: Vec = Vec::new(); + + for _ in 0..times_to_poll { + let Some(next_message_result) = consumer.poll(Duration::from_secs(2)) else { + continue; + }; + + let Ok(borrowed_next_message) = next_message_result else { + panic!( + "could not get next message from based_consumer: {}", + next_message_result.unwrap_err() + ); + }; + borrowed_messages.push(borrowed_next_message); + tokio::time::sleep(Duration::from_millis(100)).await; + } + + Ok(borrowed_messages) +} diff --git a/tests/utils/containers.rs b/tests/utils/containers.rs index f79a195f8..e5a13061b 100644 --- a/tests/utils/containers.rs +++ b/tests/utils/containers.rs @@ -28,6 +28,24 @@ impl KafkaContext { .context("Failed to initialize Kafka shared instance") .map(Arc::clone) } + + pub async fn std_out(&self) -> anyhow::Result { + let std_out_byte_vec = self + .kafka_node + .stdout_to_vec() + .await + .context("Failed to get stdout")?; + Ok(String::from_utf8(std_out_byte_vec)?) + } + + pub async fn std_err(&self) -> anyhow::Result { + let std_err_byte_vec = self + .kafka_node + .stderr_to_vec() + .await + .context("Failed to get stderr")?; + Ok(String::from_utf8(std_err_byte_vec)?) + } } async fn init() -> anyhow::Result> { diff --git a/tests/utils/producer.rs b/tests/utils/producer.rs index d553edcdd..46e8864f9 100644 --- a/tests/utils/producer.rs +++ b/tests/utils/producer.rs @@ -1,6 +1,22 @@ +use anyhow::Context; use rdkafka::config::FromClientConfig; -use rdkafka::producer::BaseProducer; +use rdkafka::producer::{BaseProducer, Producer}; +use rdkafka::util::Timeout; use rdkafka::ClientConfig; +use std::time::Duration; + +pub async fn create_producer(bootstrap_servers: &str) -> anyhow::Result { + let mut producer_client_config = ClientConfig::default(); + producer_client_config.set("bootstrap.servers", bootstrap_servers); + let base_producer_result = create_base_producer(&producer_client_config); + let Ok(base_producer) = base_producer_result else { + panic!( + "could not create based_producer: {}", + base_producer_result.unwrap_err() + ); + }; + Ok(base_producer) +} pub fn create_base_producer(config: &ClientConfig) -> anyhow::Result { let base_producer_result = BaseProducer::from_config(config); @@ -10,6 +26,14 @@ pub fn create_base_producer(config: &ClientConfig) -> anyhow::Result anyhow::Result<()> { + for _ in 0..5 { + base_producer.poll(Duration::from_millis(100)); + } + base_producer + .flush(Timeout::After(Duration::from_secs(10))) + .context("flush failed") +} diff --git a/tests/utils/rand.rs b/tests/utils/rand.rs index 495d25ba0..fabe6e668 100644 --- a/tests/utils/rand.rs +++ b/tests/utils/rand.rs @@ -2,7 +2,7 @@ use rand::distr::{Alphanumeric, SampleString}; pub fn rand_test_topic(test_name: &str) -> String { let id = Alphanumeric.sample_string(&mut rand::rng(), 10); - format!("__{}_{}", test_name, id) + format!("{}_{}", test_name, id) } pub fn rand_test_group() -> String { From b6d9bc9c41cfef460d78275412b95340ead0abf5 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 10 Oct 2025 20:45:07 -0400 Subject: [PATCH 09/56] clear up --- tests/producer.rs | 12 ++++-------- tests/utils/producer.rs | 19 +++++++++++++++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/producer.rs b/tests/producer.rs index 7987d2ecc..6b78a7a8e 100644 --- a/tests/producer.rs +++ b/tests/producer.rs @@ -2,7 +2,7 @@ use crate::utils::admin::{create_admin_client, create_topic}; use crate::utils::consumer::{create_consumer, poll_x_times_for_messages}; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; -use crate::utils::producer::{create_producer, poll_and_flush}; +use crate::utils::producer::{create_producer, send_record}; use crate::utils::rand::rand_test_topic; use rdkafka::producer::BaseRecord; use rdkafka::Message; @@ -57,13 +57,9 @@ pub async fn test_basic_produce() { let record = BaseRecord::to(&test_topic_name) // destination topic .key(&[1, 2, 3, 4]) // message key .payload("content"); // message payload - - let send_result = base_producer.send(record); - if send_result.is_err() { - panic!("could not produce record: {:?}", send_result.unwrap_err()); - } - if poll_and_flush(&base_producer).is_err() { - panic!("could not poll and flush base producer") + let send_record_result = send_record(&base_producer, record).await; + if send_record_result.is_err() { + panic!("could not send record: {}", send_record_result.unwrap_err()); } let messages_result = poll_x_times_for_messages(&consumer, 10).await; diff --git a/tests/utils/producer.rs b/tests/utils/producer.rs index 46e8864f9..6f8bfe05a 100644 --- a/tests/utils/producer.rs +++ b/tests/utils/producer.rs @@ -1,6 +1,6 @@ -use anyhow::Context; +use anyhow::{bail, Context}; use rdkafka::config::FromClientConfig; -use rdkafka::producer::{BaseProducer, Producer}; +use rdkafka::producer::{BaseProducer, BaseRecord, Producer}; use rdkafka::util::Timeout; use rdkafka::ClientConfig; use std::time::Duration; @@ -29,6 +29,21 @@ pub fn create_base_producer(config: &ClientConfig) -> anyhow::Result, +) -> anyhow::Result<()> { + let send_result = producer.send(record); + if send_result.is_err() { + bail!("could not produce record: {:?}", send_result.unwrap_err()); + } + if poll_and_flush(&producer).is_err() { + bail!("could not poll and flush base producer") + }; + + Ok(()) +} + pub fn poll_and_flush(base_producer: &BaseProducer) -> anyhow::Result<()> { for _ in 0..5 { base_producer.poll(Duration::from_millis(100)); From 180ec7e5635651e4183cd960929f5adac28dcb57 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Sat, 11 Oct 2025 15:14:56 -0400 Subject: [PATCH 10/56] test_topic_create_and_delete --- tests/admin.rs | 293 ++++++++++++++++++++++++++++++++++++++ tests/producer.rs | 5 +- tests/test_admin.rs | 236 ------------------------------ tests/utils/consumer.rs | 79 ++++++---- tests/utils/containers.rs | 10 +- tests/utils/mod.rs | 37 ++--- 6 files changed, 371 insertions(+), 289 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index e57e11442..038a84cee 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -2,6 +2,12 @@ use crate::utils::admin::create_topic; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; use crate::utils::rand::rand_test_topic; +use crate::utils::{get_broker_version, KafkaVersion}; +use rdkafka::admin::{ + AdminOptions, ConfigEntry, ConfigSource, NewPartitions, NewTopic, ResourceSpecifier, + TopicReplication, +}; +use std::time::Duration; #[path = "utils/mod.rs"] mod utils; @@ -37,3 +43,290 @@ pub async fn test_topic_creation() { ); }; } + +#[tokio::test] +async fn test_topics() { + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); + + // Create consumer client + let consumer_client = + utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers) + .await + .expect("could not create consumer client"); + + // // Verify that incorrect replication configurations are ignored when + // // creating topics. + // { + // let topic = NewTopic::new("ignored", 1, TopicReplication::Variable(&[&[0], &[0]])); + // let res = admin_client.create_topics(&[topic], &opts).await; + // assert_eq!( + // Err(KafkaError::AdminOpCreation( + // "replication configuration for topic 'ignored' assigns 2 partition(s), \ + // which does not match the specified number of partitions (1)" + // .into() + // )), + // res, + // ) + // } + // + // // Verify that incorrect replication configurations are ignored when + // // creating partitions. + // { + // let name = rand_test_topic("test_topics"); + // let topic = NewTopic::new(&name, 1, TopicReplication::Fixed(1)); + // + // let res = admin_client + // .create_topics(vec![&topic], &opts) + // .await + // .expect("topic creation failed"); + // assert_eq!(res, &[Ok(name.clone())]); + // let _ = fetch_metadata(&name); + // + // // This partition specification is obviously garbage, and so trips + // // a client-side error. + // let partitions = NewPartitions::new(&name, 2).assign(&[&[0], &[0], &[0]]); + // let res = admin_client.create_partitions(&[partitions], &opts).await; + // assert_eq!( + // res, + // Err(KafkaError::AdminOpCreation(format!( + // "partition assignment for topic '{}' assigns 3 partition(s), \ + // which is more than the requested total number of partitions (2)", + // name + // ))) + // ); + // + // // Only the server knows that this partition specification is garbage. + // let partitions = NewPartitions::new(&name, 2).assign(&[&[0], &[0]]); + // let res = admin_client + // .create_partitions(&[partitions], &opts) + // .await + // .expect("partition creation failed"); + // assert_eq!( + // res, + // &[Err((name, RDKafkaErrorCode::InvalidReplicaAssignment))], + // ); + // } + // + // // Verify that deleting a non-existent topic fails. + // { + // let name = rand_test_topic("test_topics"); + // let res = admin_client + // .delete_topics(&[&name], &opts) + // .await + // .expect("delete topics failed"); + // assert_eq!( + // res, + // &[Err((name, RDKafkaErrorCode::UnknownTopicOrPartition))] + // ); + // } + // + // // Verify that mixed-success operations properly report the successful and + // // failing operators. + // { + // let name1 = rand_test_topic("test_topics"); + // let name2 = rand_test_topic("test_topics"); + // + // let topic1 = NewTopic::new(&name1, 1, TopicReplication::Fixed(1)); + // let topic2 = NewTopic::new(&name2, 1, TopicReplication::Fixed(1)); + // + // let res = admin_client + // .create_topics(vec![&topic1], &opts) + // .await + // .expect("topic creation failed"); + // assert_eq!(res, &[Ok(name1.clone())]); + // let _ = fetch_metadata(&name1); + // + // let res = admin_client + // .create_topics(vec![&topic1, &topic2], &opts) + // .await + // .expect("topic creation failed"); + // assert_eq!( + // res, + // &[ + // Err((name1.clone(), RDKafkaErrorCode::TopicAlreadyExists)), + // Ok(name2.clone()) + // ] + // ); + // let _ = fetch_metadata(&name2); + // + // let res = admin_client + // .delete_topics(&[&name1], &opts) + // .await + // .expect("topic deletion failed"); + // assert_eq!(res, &[Ok(name1.clone())]); + // verify_delete(&name1); + // + // let res = admin_client + // .delete_topics(&[&name2, &name1], &opts) + // .await + // .expect("topic deletion failed"); + // assert_eq!( + // res, + // &[ + // Ok(name2.clone()), + // Err((name1.clone(), RDKafkaErrorCode::UnknownTopicOrPartition)) + // ] + // ); + // } +} + +#[tokio::test] +pub async fn test_topic_create_and_delete() { + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); + + // Create consumer client + let consumer_client = + utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers) + .await + .expect("could not create consumer client"); + + let topic_name_1 = rand_test_topic("test_topics"); + let topic_name_2 = rand_test_topic("test_topics"); + let topic1 = NewTopic::new(&topic_name_1, 1, TopicReplication::Fixed(1)) + .set("max.message.bytes", "1234"); + let topic2 = NewTopic { + name: &topic_name_2, + num_partitions: 3, + replication: TopicReplication::Variable(&[&[1], &[1], &[1]]), + config: Vec::new(), + }; + + // Topics created + let topic_results = admin_client + .create_topics(&[topic1, topic2], &opts) + .await + .expect("topic creation failed"); + assert_eq!( + topic_results, + &[Ok(topic_name_1.clone()), Ok(topic_name_2.clone())] + ); + + // Verify metadata + let metadata1 = utils::consumer::fetch_consumer_metadata(&consumer_client, &topic_name_1) + .expect(&format!("failed to fetch metadata for {}", &topic_name_1)); + let metadata2 = utils::consumer::fetch_consumer_metadata(&consumer_client, &topic_name_2) + .expect(&format!("failed to fetch metadata for {}", topic_name_2)); + assert_eq!(1, metadata1.topics().len()); + assert_eq!(1, metadata2.topics().len()); + let metadata_topic1 = &metadata1.topics()[0]; + let metadata_topic2 = &metadata2.topics()[0]; + assert_eq!(&topic_name_1, metadata_topic1.name()); + assert_eq!(&topic_name_2, metadata_topic2.name()); + assert_eq!(1, metadata_topic1.partitions().len()); + assert_eq!(3, metadata_topic2.partitions().len()); + + // Verifying topic configurations + let config_resource_results = admin_client + .describe_configs( + &[ + ResourceSpecifier::Topic(&topic_name_1), + ResourceSpecifier::Topic(&topic_name_2), + ], + &opts, + ) + .await + .expect("could not describe configs"); + let topic_config1 = &config_resource_results[0] + .as_ref() + .expect(&format!("failed to describe config for {}", &topic_name_1)); + let topic_config2 = &config_resource_results[1] + .as_ref() + .expect(&format!("failed to describe config for {}", &topic_name_2)); + let mut expected_entry1 = ConfigEntry { + name: "max.message.bytes".into(), + value: Some("1234".into()), + source: ConfigSource::DynamicTopic, + is_read_only: false, + is_default: false, + is_sensitive: false, + }; + let default_max_msg_bytes = if get_broker_version(&kafka_context) <= KafkaVersion(2, 3, 0, 0) { + "1000012" + } else { + "1048588" + }; + let expected_entry2 = ConfigEntry { + name: "max.message.bytes".into(), + value: Some(default_max_msg_bytes.into()), + source: ConfigSource::Default, + is_read_only: false, + is_default: true, + is_sensitive: false, + }; + if get_broker_version(&kafka_context) < KafkaVersion(1, 1, 0, 0) { + expected_entry1.source = ConfigSource::Unknown; + } + assert_eq!( + Some(&expected_entry1), + topic_config1.get("max.message.bytes") + ); + assert_eq!( + Some(&expected_entry2), + topic_config2.get("max.message.bytes") + ); + let config_entries1 = topic_config1.entry_map(); + let config_entries2 = topic_config2.entry_map(); + assert_eq!(topic_config1.entries.len(), config_entries1.len()); + assert_eq!(topic_config2.entries.len(), config_entries2.len()); + assert_eq!( + Some(&&expected_entry1), + config_entries1.get("max.message.bytes") + ); + assert_eq!( + Some(&&expected_entry2), + config_entries2.get("max.message.bytes") + ); + + let partitions1 = NewPartitions::new(&topic_name_1, 5); + let res = admin_client + .create_partitions(&[partitions1], &opts) + .await + .expect("partition creation failed"); + assert_eq!(res, &[Ok(topic_name_1.clone())]); + + let mut tries = 0; + loop { + let metadata = utils::consumer::fetch_consumer_metadata(&consumer_client, &topic_name_1) + .expect(&format!("failed to fetch metadata for {}", &topic_name_1)); + let topic = &metadata.topics()[0]; + let n = topic.partitions().len(); + if n == 5 { + break; + } else if tries >= 5 { + panic!("topic has {} partitions, but expected {}", n, 5); + } else { + tries += 1; + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + + let res = admin_client + .delete_topics(&[&topic_name_1, &topic_name_2], &opts) + .await + .expect("topic deletion failed"); + assert_eq!(res, &[Ok(topic_name_1.clone()), Ok(topic_name_2.clone())]); + utils::consumer::verify_topic_deleted(&consumer_client, &topic_name_1) + .expect(&format!("could not delete topic for {}", &topic_name_1)); + utils::consumer::verify_topic_deleted(&consumer_client, &topic_name_2) + .expect(&format!("could not delete topic for {}", &topic_name_2)); +} diff --git a/tests/producer.rs b/tests/producer.rs index 6b78a7a8e..a6bb1580e 100644 --- a/tests/producer.rs +++ b/tests/producer.rs @@ -1,5 +1,5 @@ use crate::utils::admin::{create_admin_client, create_topic}; -use crate::utils::consumer::{create_consumer, poll_x_times_for_messages}; +use crate::utils::consumer::{create_subscribed_base_consumer, poll_x_times_for_messages}; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; use crate::utils::producer::{create_producer, send_record}; @@ -38,7 +38,8 @@ pub async fn test_basic_produce() { ); } - let consumer_result = create_consumer(&kafka_context.bootstrap_servers, &test_topic_name).await; + let consumer_result = + create_subscribed_base_consumer(&kafka_context.bootstrap_servers, &test_topic_name).await; let Ok(consumer) = consumer_result else { panic!( "could not create consumer: {}", diff --git a/tests/test_admin.rs b/tests/test_admin.rs index a44921f5b..5d5a90975 100644 --- a/tests/test_admin.rs +++ b/tests/test_admin.rs @@ -119,242 +119,6 @@ fn verify_delete(topic: &str) { .unwrap() } -#[tokio::test] -async fn test_topics() { - let admin_client = create_admin_client(); - let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); - - // Verify that topics are created as specified, and that they can later - // be deleted. - { - let name1 = rand_test_topic("test_topics"); - let name2 = rand_test_topic("test_topics"); - - // Test both the builder API and the literal construction. - let topic1 = - NewTopic::new(&name1, 1, TopicReplication::Fixed(1)).set("max.message.bytes", "1234"); - let topic2 = NewTopic { - name: &name2, - num_partitions: 3, - replication: TopicReplication::Variable(&[&[0], &[0], &[0]]), - config: Vec::new(), - }; - - let res = admin_client - .create_topics(&[topic1, topic2], &opts) - .await - .expect("topic creation failed"); - assert_eq!(res, &[Ok(name1.clone()), Ok(name2.clone())]); - - let metadata1 = fetch_metadata(&name1); - let metadata2 = fetch_metadata(&name2); - assert_eq!(1, metadata1.topics().len()); - assert_eq!(1, metadata2.topics().len()); - let metadata_topic1 = &metadata1.topics()[0]; - let metadata_topic2 = &metadata2.topics()[0]; - assert_eq!(&name1, metadata_topic1.name()); - assert_eq!(&name2, metadata_topic2.name()); - assert_eq!(1, metadata_topic1.partitions().len()); - assert_eq!(3, metadata_topic2.partitions().len()); - - let res = admin_client - .describe_configs( - &[ - ResourceSpecifier::Topic(&name1), - ResourceSpecifier::Topic(&name2), - ], - &opts, - ) - .await - .expect("describe configs failed"); - let config1 = &res[0].as_ref().expect("describe configs failed on topic 1"); - let config2 = &res[1].as_ref().expect("describe configs failed on topic 2"); - let mut expected_entry1 = ConfigEntry { - name: "max.message.bytes".into(), - value: Some("1234".into()), - source: ConfigSource::DynamicTopic, - is_read_only: false, - is_default: false, - is_sensitive: false, - }; - let default_max_msg_bytes = if get_broker_version() <= KafkaVersion(2, 3, 0, 0) { - "1000012" - } else { - "1048588" - }; - let expected_entry2 = ConfigEntry { - name: "max.message.bytes".into(), - value: Some(default_max_msg_bytes.into()), - source: ConfigSource::Default, - is_read_only: false, - is_default: true, - is_sensitive: false, - }; - if get_broker_version() < KafkaVersion(1, 1, 0, 0) { - expected_entry1.source = ConfigSource::Unknown; - } - assert_eq!(Some(&expected_entry1), config1.get("max.message.bytes")); - assert_eq!(Some(&expected_entry2), config2.get("max.message.bytes")); - let config_entries1 = config1.entry_map(); - let config_entries2 = config2.entry_map(); - assert_eq!(config1.entries.len(), config_entries1.len()); - assert_eq!(config2.entries.len(), config_entries2.len()); - assert_eq!( - Some(&&expected_entry1), - config_entries1.get("max.message.bytes") - ); - assert_eq!( - Some(&&expected_entry2), - config_entries2.get("max.message.bytes") - ); - - let partitions1 = NewPartitions::new(&name1, 5); - let res = admin_client - .create_partitions(&[partitions1], &opts) - .await - .expect("partition creation failed"); - assert_eq!(res, &[Ok(name1.clone())]); - - let mut tries = 0; - loop { - let metadata = fetch_metadata(&name1); - let topic = &metadata.topics()[0]; - let n = topic.partitions().len(); - if n == 5 { - break; - } else if tries >= 5 { - panic!("topic has {} partitions, but expected {}", n, 5); - } else { - tries += 1; - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - - let res = admin_client - .delete_topics(&[&name1, &name2], &opts) - .await - .expect("topic deletion failed"); - assert_eq!(res, &[Ok(name1.clone()), Ok(name2.clone())]); - verify_delete(&name1); - verify_delete(&name2); - } - - // Verify that incorrect replication configurations are ignored when - // creating topics. - { - let topic = NewTopic::new("ignored", 1, TopicReplication::Variable(&[&[0], &[0]])); - let res = admin_client.create_topics(&[topic], &opts).await; - assert_eq!( - Err(KafkaError::AdminOpCreation( - "replication configuration for topic 'ignored' assigns 2 partition(s), \ - which does not match the specified number of partitions (1)" - .into() - )), - res, - ) - } - - // Verify that incorrect replication configurations are ignored when - // creating partitions. - { - let name = rand_test_topic("test_topics"); - let topic = NewTopic::new(&name, 1, TopicReplication::Fixed(1)); - - let res = admin_client - .create_topics(vec![&topic], &opts) - .await - .expect("topic creation failed"); - assert_eq!(res, &[Ok(name.clone())]); - let _ = fetch_metadata(&name); - - // This partition specification is obviously garbage, and so trips - // a client-side error. - let partitions = NewPartitions::new(&name, 2).assign(&[&[0], &[0], &[0]]); - let res = admin_client.create_partitions(&[partitions], &opts).await; - assert_eq!( - res, - Err(KafkaError::AdminOpCreation(format!( - "partition assignment for topic '{}' assigns 3 partition(s), \ - which is more than the requested total number of partitions (2)", - name - ))) - ); - - // Only the server knows that this partition specification is garbage. - let partitions = NewPartitions::new(&name, 2).assign(&[&[0], &[0]]); - let res = admin_client - .create_partitions(&[partitions], &opts) - .await - .expect("partition creation failed"); - assert_eq!( - res, - &[Err((name, RDKafkaErrorCode::InvalidReplicaAssignment))], - ); - } - - // Verify that deleting a non-existent topic fails. - { - let name = rand_test_topic("test_topics"); - let res = admin_client - .delete_topics(&[&name], &opts) - .await - .expect("delete topics failed"); - assert_eq!( - res, - &[Err((name, RDKafkaErrorCode::UnknownTopicOrPartition))] - ); - } - - // Verify that mixed-success operations properly report the successful and - // failing operators. - { - let name1 = rand_test_topic("test_topics"); - let name2 = rand_test_topic("test_topics"); - - let topic1 = NewTopic::new(&name1, 1, TopicReplication::Fixed(1)); - let topic2 = NewTopic::new(&name2, 1, TopicReplication::Fixed(1)); - - let res = admin_client - .create_topics(vec![&topic1], &opts) - .await - .expect("topic creation failed"); - assert_eq!(res, &[Ok(name1.clone())]); - let _ = fetch_metadata(&name1); - - let res = admin_client - .create_topics(vec![&topic1, &topic2], &opts) - .await - .expect("topic creation failed"); - assert_eq!( - res, - &[ - Err((name1.clone(), RDKafkaErrorCode::TopicAlreadyExists)), - Ok(name2.clone()) - ] - ); - let _ = fetch_metadata(&name2); - - let res = admin_client - .delete_topics(&[&name1], &opts) - .await - .expect("topic deletion failed"); - assert_eq!(res, &[Ok(name1.clone())]); - verify_delete(&name1); - - let res = admin_client - .delete_topics(&[&name2, &name1], &opts) - .await - .expect("topic deletion failed"); - assert_eq!( - res, - &[ - Ok(name2.clone()), - Err((name1.clone(), RDKafkaErrorCode::UnknownTopicOrPartition)) - ] - ); - } -} - /// Test the admin client's delete records functionality. #[tokio::test] async fn test_delete_records() { diff --git a/tests/utils/consumer.rs b/tests/utils/consumer.rs index 267b1e212..cccb08d4e 100644 --- a/tests/utils/consumer.rs +++ b/tests/utils/consumer.rs @@ -1,15 +1,27 @@ use crate::utils::consumer; use crate::utils::rand::rand_test_group; -use anyhow::Context; +use anyhow::{bail, Context}; +use backon::{BlockingRetryable, ExponentialBuilder}; use rdkafka::config::FromClientConfig; use rdkafka::consumer::{BaseConsumer, Consumer}; use rdkafka::message::BorrowedMessage; +use rdkafka::metadata::Metadata; use rdkafka::ClientConfig; use std::time::Duration; -pub async fn create_consumer( +pub async fn create_subscribed_base_consumer( bootstrap_servers: &str, test_topic: &str, +) -> anyhow::Result { + let unsubscribed_base_consumer = create_unsubscribed_base_consumer(bootstrap_servers).await?; + unsubscribed_base_consumer + .subscribe(&[test_topic]) + .context("Failed to subscribe to topic")?; + Ok(unsubscribed_base_consumer) +} + +pub async fn create_unsubscribed_base_consumer( + bootstrap_servers: &str, ) -> anyhow::Result { let mut consumer_client_config = ClientConfig::default(); consumer_client_config.set("group.id", rand_test_group()); @@ -21,29 +33,7 @@ pub async fn create_consumer( consumer_client_config.set("debug", "all"); consumer_client_config.set("auto.offset.reset", "earliest"); - let base_consumer_result = - consumer::create_subscribed_base(consumer_client_config, &[&test_topic]).await; - let Ok(base_consumer) = base_consumer_result else { - panic!( - "could not create base consumer: {}", - base_consumer_result.unwrap_err() - ) - }; - - Ok(base_consumer) -} - -pub async fn create_subscribed_base( - client_config: ClientConfig, - topics: &[&str], -) -> anyhow::Result { - let base_consumer = - BaseConsumer::from_config(&client_config).context("Failed to create consumer")?; - base_consumer - .subscribe(topics) - .context("Failed to subscribe to topic")?; - - Ok(base_consumer) + BaseConsumer::from_config(&consumer_client_config).context("Failed to create consumer") } pub async fn poll_x_times_for_messages( @@ -69,3 +59,42 @@ pub async fn poll_x_times_for_messages( Ok(borrowed_messages) } + +pub fn fetch_consumer_metadata(consumer: &BaseConsumer, topic: &str) -> anyhow::Result { + let timeout = Some(Duration::from_secs(1)); + + (|| { + let metadata = consumer + .fetch_metadata(Some(topic), timeout) + .context("Failed to fetch metadata")?; + if metadata.topics().is_empty() { + bail!("metadata fetch returned no topics".to_string()) + } + let topic = &metadata.topics()[0]; + if topic.partitions().is_empty() { + bail!("metadata fetch returned a topic with no partitions".to_string()) + } + Ok(metadata) + }) + .retry(ExponentialBuilder::default().with_max_delay(Duration::from_secs(5))) + .call() +} + +pub fn verify_topic_deleted(consumer: &BaseConsumer, topic: &str) -> anyhow::Result<()> { + let timeout = Some(Duration::from_secs(1)); + + (|| { + // Asking about the topic specifically will recreate it (under the + // default Kafka configuration, at least) so we have to ask for the list + // of all topics and search through it. + let metadata = consumer + .fetch_metadata(None, timeout) + .context("Failed to fetch metadata")?; + if metadata.topics().iter().any(|t| t.name() == topic) { + bail!(format!("topic {} still exists", topic)) + } + Ok(()) + }) + .retry(ExponentialBuilder::default().with_max_delay(Duration::from_secs(5))) + .call() +} diff --git a/tests/utils/containers.rs b/tests/utils/containers.rs index e5a13061b..560d7ce7e 100644 --- a/tests/utils/containers.rs +++ b/tests/utils/containers.rs @@ -4,12 +4,13 @@ use std::sync::Arc; use testcontainers_modules::kafka::apache::Kafka; use testcontainers_modules::testcontainers::core::ContainerPort; use testcontainers_modules::testcontainers::runners::AsyncRunner; -use testcontainers_modules::testcontainers::ContainerAsync; +use testcontainers_modules::testcontainers::{ContainerAsync, Image}; use tokio::sync::OnceCell; pub struct KafkaContext { kafka_node: ContainerAsync, pub bootstrap_servers: String, + pub version: String, } impl Debug for KafkaContext { @@ -49,11 +50,13 @@ impl KafkaContext { } async fn init() -> anyhow::Result> { - let kafka_node = Kafka::default() + let kafka_container = Kafka::default(); + let kafka_version = kafka_container.tag().to_string(); + + let kafka_node = kafka_container .start() .await .context("Failed to start Kafka")?; - let kafka_host = kafka_node .get_host() .await @@ -65,6 +68,7 @@ async fn init() -> anyhow::Result> { Ok::, anyhow::Error>(Arc::new(KafkaContext { kafka_node, bootstrap_servers: format!("{}:{}", kafka_host, kafka_port), + version: kafka_version, })) } diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index 3522ebc87..087c30c69 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -8,11 +8,12 @@ pub mod producer; pub mod rand; use std::collections::HashMap; -use std::env::{self, VarError}; +use std::env::{self}; use std::time::Duration; use regex::Regex; +use crate::utils::containers::KafkaContext; use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication}; use rdkafka::client::ClientContext; use rdkafka::config::ClientConfig; @@ -27,29 +28,19 @@ pub fn get_bootstrap_server() -> String { env::var("KAFKA_HOST").unwrap_or_else(|_| "localhost:9092".to_owned()) } -pub fn get_broker_version() -> KafkaVersion { - // librdkafka doesn't expose this directly, sadly. - match env::var("KAFKA_VERSION") { - Ok(v) => { - let regex = Regex::new(r"^(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?$").unwrap(); - match regex.captures(&v) { - Some(captures) => { - let extract = |i| { - captures - .get(i) - .map(|m| m.as_str().parse().unwrap()) - .unwrap_or(0) - }; - KafkaVersion(extract(1), extract(2), extract(3), extract(4)) - } - None => panic!("KAFKA_VERSION env var was not in expected [n[.n[.n[.n]]]] format"), - } - } - Err(VarError::NotUnicode(_)) => { - panic!("KAFKA_VERSION env var contained non-unicode characters") +pub fn get_broker_version(kafka_context: &KafkaContext) -> KafkaVersion { + let regex = Regex::new(r"^(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?$").unwrap(); + match regex.captures(&kafka_context.version) { + Some(captures) => { + let extract = |i| { + captures + .get(i) + .map(|m| m.as_str().parse().unwrap()) + .unwrap_or(0) + }; + KafkaVersion(extract(1), extract(2), extract(3), extract(4)) } - // If the environment variable is unset, assume we're running the latest version. - Err(VarError::NotPresent) => KafkaVersion(u32::MAX, u32::MAX, u32::MAX, u32::MAX), + None => panic!("KAFKA_VERSION env var was not in expected [n[.n[.n[.n]]]] format"), } } From c06b57e30d439afdcefae58d5b9f311506d5681a Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Sat, 11 Oct 2025 20:48:59 -0400 Subject: [PATCH 11/56] fix: test_incorect_replication_factors_are_ignored --- tests/admin.rs | 62 ++++++++++++++++++++++++++++++++++------------ tests/utils/mod.rs | 2 ++ 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index 038a84cee..05d9f6e0f 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -7,6 +7,7 @@ use rdkafka::admin::{ AdminOptions, ConfigEntry, ConfigSource, NewPartitions, NewTopic, ResourceSpecifier, TopicReplication, }; +use rdkafka::error::KafkaError; use std::time::Duration; #[path = "utils/mod.rs"] @@ -65,21 +66,6 @@ async fn test_topics() { .await .expect("could not create consumer client"); - // // Verify that incorrect replication configurations are ignored when - // // creating topics. - // { - // let topic = NewTopic::new("ignored", 1, TopicReplication::Variable(&[&[0], &[0]])); - // let res = admin_client.create_topics(&[topic], &opts).await; - // assert_eq!( - // Err(KafkaError::AdminOpCreation( - // "replication configuration for topic 'ignored' assigns 2 partition(s), \ - // which does not match the specified number of partitions (1)" - // .into() - // )), - // res, - // ) - // } - // // // Verify that incorrect replication configurations are ignored when // // creating partitions. // { @@ -207,7 +193,11 @@ pub async fn test_topic_create_and_delete() { let topic2 = NewTopic { name: &topic_name_2, num_partitions: 3, - replication: TopicReplication::Variable(&[&[1], &[1], &[1]]), + replication: TopicReplication::Variable(&[ + &[utils::BROKER_ID], + &[utils::BROKER_ID], + &[utils::BROKER_ID], + ]), config: Vec::new(), }; @@ -330,3 +320,43 @@ pub async fn test_topic_create_and_delete() { utils::consumer::verify_topic_deleted(&consumer_client, &topic_name_2) .expect(&format!("could not delete topic for {}", &topic_name_2)); } + +/// Verify that incorrect replication configurations are ignored when +/// creating topics. +#[tokio::test] +pub async fn test_incorect_replication_factors_are_ignored() { + // Get Kafka container context. + let kafka_context_result = KafkaContext::shared().await; + let Ok(kafka_context) = kafka_context_result else { + panic!( + "could not create kafka context: {}", + kafka_context_result.unwrap_err() + ); + }; + let test_topic_name = rand_test_topic("testing-topic"); + + let admin_client_result = + utils::admin::create_admin_client(&kafka_context.bootstrap_servers).await; + let Ok(admin_client) = admin_client_result else { + panic!( + "could not create admin client: {}", + admin_client_result.unwrap_err() + ); + }; + let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); + + let topic = NewTopic::new( + "ignored", + 1, + TopicReplication::Variable(&[&[utils::BROKER_ID], &[utils::BROKER_ID]]), + ); + let res = admin_client.create_topics(&[topic], &opts).await; + assert_eq!( + Err(KafkaError::AdminOpCreation( + "replication configuration for topic 'ignored' assigns 2 partition(s), \ + which does not match the specified number of partitions (1)" + .into() + )), + res, + ) +} diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index 087c30c69..06e1a0c7e 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -24,6 +24,8 @@ use rdkafka::producer::{FutureProducer, FutureRecord}; use rdkafka::statistics::Statistics; use rdkafka::TopicPartitionList; +pub const BROKER_ID: i32 = 1; + pub fn get_bootstrap_server() -> String { env::var("KAFKA_HOST").unwrap_or_else(|_| "localhost:9092".to_owned()) } From 87f159b2de9c617d1ffc894ccb7e9f9dc0f4f7c6 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Sat, 11 Oct 2025 21:00:30 -0400 Subject: [PATCH 12/56] fix: test_incorrect_replication_factors_are_ignored_when_creating_partitions --- tests/admin.rs | 104 ++++++++++++++++++++++++++++----------------- tests/utils/mod.rs | 21 --------- 2 files changed, 66 insertions(+), 59 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index 05d9f6e0f..b0ee393a8 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -9,6 +9,7 @@ use rdkafka::admin::{ }; use rdkafka::error::KafkaError; use std::time::Duration; +use rdkafka_sys::RDKafkaErrorCode; #[path = "utils/mod.rs"] mod utils; @@ -66,43 +67,6 @@ async fn test_topics() { .await .expect("could not create consumer client"); - // // Verify that incorrect replication configurations are ignored when - // // creating partitions. - // { - // let name = rand_test_topic("test_topics"); - // let topic = NewTopic::new(&name, 1, TopicReplication::Fixed(1)); - // - // let res = admin_client - // .create_topics(vec![&topic], &opts) - // .await - // .expect("topic creation failed"); - // assert_eq!(res, &[Ok(name.clone())]); - // let _ = fetch_metadata(&name); - // - // // This partition specification is obviously garbage, and so trips - // // a client-side error. - // let partitions = NewPartitions::new(&name, 2).assign(&[&[0], &[0], &[0]]); - // let res = admin_client.create_partitions(&[partitions], &opts).await; - // assert_eq!( - // res, - // Err(KafkaError::AdminOpCreation(format!( - // "partition assignment for topic '{}' assigns 3 partition(s), \ - // which is more than the requested total number of partitions (2)", - // name - // ))) - // ); - // - // // Only the server knows that this partition specification is garbage. - // let partitions = NewPartitions::new(&name, 2).assign(&[&[0], &[0]]); - // let res = admin_client - // .create_partitions(&[partitions], &opts) - // .await - // .expect("partition creation failed"); - // assert_eq!( - // res, - // &[Err((name, RDKafkaErrorCode::InvalidReplicaAssignment))], - // ); - // } // // // Verify that deleting a non-existent topic fails. // { @@ -324,7 +288,7 @@ pub async fn test_topic_create_and_delete() { /// Verify that incorrect replication configurations are ignored when /// creating topics. #[tokio::test] -pub async fn test_incorect_replication_factors_are_ignored() { +pub async fn test_incorrect_replication_factors_are_ignored_when_creating_topics() { // Get Kafka container context. let kafka_context_result = KafkaContext::shared().await; let Ok(kafka_context) = kafka_context_result else { @@ -360,3 +324,67 @@ pub async fn test_incorect_replication_factors_are_ignored() { res, ) } + +/// Verify that incorrect replication configurations are ignored when +/// creating partitions. +#[tokio::test] +pub async fn test_incorrect_replication_factors_are_ignored_when_creating_partitions() { + // Get Kafka container context. + let kafka_context_result = KafkaContext::shared().await; + let Ok(kafka_context) = kafka_context_result else { + panic!( + "could not create kafka context: {}", + kafka_context_result.unwrap_err() + ); + }; + + let admin_client_result = + utils::admin::create_admin_client(&kafka_context.bootstrap_servers).await; + let Ok(admin_client) = admin_client_result else { + panic!( + "could not create admin client: {}", + admin_client_result.unwrap_err() + ); + }; + let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); + + // Create consumer client + let consumer_client = + utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers) + .await + .expect("could not create consumer client"); + + let name = rand_test_topic("test_topics"); + let topic = NewTopic::new(&name, 1, TopicReplication::Fixed(1)); + + let res = admin_client + .create_topics(vec![&topic], &opts) + .await + .expect("topic creation failed"); + assert_eq!(res, &[Ok(name.clone())]); + let _ = utils::consumer::fetch_consumer_metadata(&consumer_client, &name); + + // This partition specification is obviously garbage, and so trips + // a client-side error. + let partitions = NewPartitions::new(&name, 2).assign(&[&[0], &[0], &[0]]); + let res = admin_client.create_partitions(&[partitions], &opts).await; + assert_eq!( + res, + Err(KafkaError::AdminOpCreation(format!( + "partition assignment for topic '{}' assigns 3 partition(s), \ + which is more than the requested total number of partitions (2)", + name + ))) + ); + + // Only the server knows that this partition specification is garbage. + let partitions = NewPartitions::new(&name, 2).assign(&[&[0], &[0]]); + let res = admin_client + .create_partitions(&[partitions], &opts) + .await + .expect("partition creation failed"); + assert_eq!( + res, + &[Err((name, RDKafkaErrorCode::InvalidReplicaAssignment))], + ); +} diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index 06e1a0c7e..27981aee2 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -177,24 +177,3 @@ pub fn consumer_config( config } - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_populate_topic() { - let topic_name = rand::rand_test_topic("test_populate_topic"); - let message_map = populate_topic(&topic_name, 100, &value_fn, &key_fn, Some(0), None).await; - - let total_messages = message_map - .iter() - .filter(|&(&(partition, _), _)| partition == 0) - .count(); - assert_eq!(total_messages, 100); - - let mut ids = message_map.values().copied().collect::>(); - ids.sort(); - assert_eq!(ids, (0..100).collect::>()); - } -} From 722deb29dfdbb412831d904c7e16c028baec52d1 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Sat, 11 Oct 2025 21:09:44 -0400 Subject: [PATCH 13/56] fix: test_mixed_success_results --- tests/admin.rs | 160 ++++++++++++++++++++++++---------------- tests/utils/consumer.rs | 1 - 2 files changed, 96 insertions(+), 65 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index b0ee393a8..ca6686b58 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -8,8 +8,8 @@ use rdkafka::admin::{ TopicReplication, }; use rdkafka::error::KafkaError; -use std::time::Duration; use rdkafka_sys::RDKafkaErrorCode; +use std::time::Duration; #[path = "utils/mod.rs"] mod utils; @@ -66,69 +66,6 @@ async fn test_topics() { utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers) .await .expect("could not create consumer client"); - - // - // // Verify that deleting a non-existent topic fails. - // { - // let name = rand_test_topic("test_topics"); - // let res = admin_client - // .delete_topics(&[&name], &opts) - // .await - // .expect("delete topics failed"); - // assert_eq!( - // res, - // &[Err((name, RDKafkaErrorCode::UnknownTopicOrPartition))] - // ); - // } - // - // // Verify that mixed-success operations properly report the successful and - // // failing operators. - // { - // let name1 = rand_test_topic("test_topics"); - // let name2 = rand_test_topic("test_topics"); - // - // let topic1 = NewTopic::new(&name1, 1, TopicReplication::Fixed(1)); - // let topic2 = NewTopic::new(&name2, 1, TopicReplication::Fixed(1)); - // - // let res = admin_client - // .create_topics(vec![&topic1], &opts) - // .await - // .expect("topic creation failed"); - // assert_eq!(res, &[Ok(name1.clone())]); - // let _ = fetch_metadata(&name1); - // - // let res = admin_client - // .create_topics(vec![&topic1, &topic2], &opts) - // .await - // .expect("topic creation failed"); - // assert_eq!( - // res, - // &[ - // Err((name1.clone(), RDKafkaErrorCode::TopicAlreadyExists)), - // Ok(name2.clone()) - // ] - // ); - // let _ = fetch_metadata(&name2); - // - // let res = admin_client - // .delete_topics(&[&name1], &opts) - // .await - // .expect("topic deletion failed"); - // assert_eq!(res, &[Ok(name1.clone())]); - // verify_delete(&name1); - // - // let res = admin_client - // .delete_topics(&[&name2, &name1], &opts) - // .await - // .expect("topic deletion failed"); - // assert_eq!( - // res, - // &[ - // Ok(name2.clone()), - // Err((name1.clone(), RDKafkaErrorCode::UnknownTopicOrPartition)) - // ] - // ); - // } } #[tokio::test] @@ -388,3 +325,98 @@ pub async fn test_incorrect_replication_factors_are_ignored_when_creating_partit &[Err((name, RDKafkaErrorCode::InvalidReplicaAssignment))], ); } + +/// Verify that deleting a non-existent topic fails. +#[tokio::test] +pub async fn test_delete_nonexistent_topics() { + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); + + let name = rand_test_topic("test_topics"); + let res = admin_client + .delete_topics(&[&name], &opts) + .await + .expect("delete topics failed"); + assert_eq!( + res, + &[Err((name, RDKafkaErrorCode::UnknownTopicOrPartition))] + ); +} + +/// Verify that mixed-success operations properly report the successful and +/// failing operators. +#[tokio::test] +pub async fn test_mixed_success_results() { + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); + + // Create consumer client + let consumer_client = + utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers) + .await + .expect("could not create consumer client"); + + let name1 = rand_test_topic("test_topics"); + let name2 = rand_test_topic("test_topics"); + + let topic1 = NewTopic::new(&name1, 1, TopicReplication::Fixed(1)); + let topic2 = NewTopic::new(&name2, 1, TopicReplication::Fixed(1)); + + let res = admin_client + .create_topics(vec![&topic1], &opts) + .await + .expect("topic creation failed"); + assert_eq!(res, &[Ok(name1.clone())]); + let _ = utils::consumer::fetch_consumer_metadata(&consumer_client, &name1) + .expect(&format!("could not fetch consumer metadata for {}", name1)); + + let res = admin_client + .create_topics(vec![&topic1, &topic2], &opts) + .await + .expect("topic creation failed"); + assert_eq!( + res, + &[ + Err((name1.clone(), RDKafkaErrorCode::TopicAlreadyExists)), + Ok(name2.clone()) + ] + ); + let _ = utils::consumer::fetch_consumer_metadata(&consumer_client, &name2) + .expect(&format!("could not fetch consumer metadata for {}", name2)); + + let res = admin_client + .delete_topics(&[&name1], &opts) + .await + .expect("topic deletion failed"); + assert_eq!(res, &[Ok(name1.clone())]); + utils::consumer::verify_topic_deleted(&consumer_client, &name1) + .expect(&format!("could not verify topic \"{}\" was deleted", name1)); + + let res = admin_client + .delete_topics(&[&name2, &name1], &opts) + .await + .expect("topic deletion failed"); + assert_eq!( + res, + &[ + Ok(name2.clone()), + Err((name1.clone(), RDKafkaErrorCode::UnknownTopicOrPartition)) + ] + ); +} diff --git a/tests/utils/consumer.rs b/tests/utils/consumer.rs index cccb08d4e..954c69417 100644 --- a/tests/utils/consumer.rs +++ b/tests/utils/consumer.rs @@ -1,4 +1,3 @@ -use crate::utils::consumer; use crate::utils::rand::rand_test_group; use anyhow::{bail, Context}; use backon::{BlockingRetryable, ExponentialBuilder}; From e9d5812edcb5e3664113fcb36fdb54c66ef8c52c Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Sat, 11 Oct 2025 21:12:54 -0400 Subject: [PATCH 14/56] fix: test commenting --- tests/admin.rs | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index ca6686b58..a41d1a76e 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -14,6 +14,7 @@ use std::time::Duration; #[path = "utils/mod.rs"] mod utils; +/// Validates thast topics can be properly created. #[tokio::test] pub async fn test_topic_creation() { init_test_logger(); @@ -46,28 +47,8 @@ pub async fn test_topic_creation() { }; } -#[tokio::test] -async fn test_topics() { - init_test_logger(); - - // Get Kafka container context. - let kafka_context = KafkaContext::shared() - .await - .expect("could not create kafka context"); - - // Create admin client - let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) - .await - .expect("could not create admin client"); - let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); - - // Create consumer client - let consumer_client = - utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers) - .await - .expect("could not create consumer client"); -} - +/// Verify that topics are created as specified, and that they can later +/// be deleted. #[tokio::test] pub async fn test_topic_create_and_delete() { // Get Kafka container context. From cab0b48822a80f5b8ff8b94d334ef50d4cfc46eb Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Sat, 11 Oct 2025 21:41:50 -0400 Subject: [PATCH 15/56] fix: test_delete_records --- tests/admin.rs | 142 +++++++++++++++++- .../base_producer.rs} | 0 tests/utils/producer/future_producer.rs | 12 ++ tests/utils/producer/mod.rs | 2 + 4 files changed, 155 insertions(+), 1 deletion(-) rename tests/utils/{producer.rs => producer/base_producer.rs} (100%) create mode 100644 tests/utils/producer/future_producer.rs create mode 100644 tests/utils/producer/mod.rs diff --git a/tests/admin.rs b/tests/admin.rs index a41d1a76e..8fe0307b5 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -3,11 +3,14 @@ use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; use crate::utils::rand::rand_test_topic; use crate::utils::{get_broker_version, KafkaVersion}; +use backon::{BlockingRetryable, ExponentialBuilder}; use rdkafka::admin::{ AdminOptions, ConfigEntry, ConfigSource, NewPartitions, NewTopic, ResourceSpecifier, TopicReplication, }; use rdkafka::error::KafkaError; +use rdkafka::producer::{FutureRecord, Producer}; +use rdkafka::{Offset, TopicPartitionList}; use rdkafka_sys::RDKafkaErrorCode; use std::time::Duration; @@ -51,6 +54,8 @@ pub async fn test_topic_creation() { /// be deleted. #[tokio::test] pub async fn test_topic_create_and_delete() { + init_test_logger(); + // Get Kafka container context. let kafka_context = KafkaContext::shared() .await @@ -207,6 +212,8 @@ pub async fn test_topic_create_and_delete() { /// creating topics. #[tokio::test] pub async fn test_incorrect_replication_factors_are_ignored_when_creating_topics() { + init_test_logger(); + // Get Kafka container context. let kafka_context_result = KafkaContext::shared().await; let Ok(kafka_context) = kafka_context_result else { @@ -215,7 +222,6 @@ pub async fn test_incorrect_replication_factors_are_ignored_when_creating_topics kafka_context_result.unwrap_err() ); }; - let test_topic_name = rand_test_topic("testing-topic"); let admin_client_result = utils::admin::create_admin_client(&kafka_context.bootstrap_servers).await; @@ -247,6 +253,8 @@ pub async fn test_incorrect_replication_factors_are_ignored_when_creating_topics /// creating partitions. #[tokio::test] pub async fn test_incorrect_replication_factors_are_ignored_when_creating_partitions() { + init_test_logger(); + // Get Kafka container context. let kafka_context_result = KafkaContext::shared().await; let Ok(kafka_context) = kafka_context_result else { @@ -310,6 +318,8 @@ pub async fn test_incorrect_replication_factors_are_ignored_when_creating_partit /// Verify that deleting a non-existent topic fails. #[tokio::test] pub async fn test_delete_nonexistent_topics() { + init_test_logger(); + // Get Kafka container context. let kafka_context = KafkaContext::shared() .await @@ -336,6 +346,8 @@ pub async fn test_delete_nonexistent_topics() { /// failing operators. #[tokio::test] pub async fn test_mixed_success_results() { + init_test_logger(); + // Get Kafka container context. let kafka_context = KafkaContext::shared() .await @@ -401,3 +413,131 @@ pub async fn test_mixed_success_results() { ] ); } + +/// Test the admin client's delete records functionality. +#[tokio::test] +async fn test_delete_records() { + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + + // Create producer client + let producer_client = + utils::producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("could not create producer_client"); + + let timeout = Some(Duration::from_secs(1)); + let opts = AdminOptions::new().operation_timeout(timeout); + let topic = rand_test_topic("test_delete_records"); + let make_record = || FutureRecord::::to(&topic).payload("data"); + + // Create a topic with a single partition. + admin_client + .create_topics( + &[NewTopic::new(&topic, 1, TopicReplication::Fixed(1))], + &opts, + ) + .await + .expect("topic creation failed"); + + // Ensure that the topic begins with low and high water marks of 0. + let (lo, hi) = (|| { + producer_client + .client() + .fetch_watermarks(&topic, 0, timeout) + }) + .retry(ExponentialBuilder::default().with_max_delay(Duration::from_secs(5))) + .call() + .unwrap(); + assert_eq!(lo, 0); + assert_eq!(hi, 0); + + // Produce five messages to the topic. + for _ in 0..5 { + producer_client.send(make_record(), timeout).await.unwrap(); + } + + // Ensure that the high water mark has advanced to 5. + let (lo, hi) = producer_client + .client() + .fetch_watermarks(&topic, 0, timeout) + .unwrap(); + assert_eq!(lo, 0); + assert_eq!(hi, 5); + + // Delete the record at offset 0. + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&topic, 0, Offset::Offset(1)) + .unwrap(); + let res_tpl = admin_client.delete_records(&tpl, &opts).await.unwrap(); + assert_eq!(res_tpl.count(), 1); + assert_eq!(res_tpl.elements()[0].topic(), topic); + assert_eq!(res_tpl.elements()[0].partition(), 0); + assert_eq!(res_tpl.elements()[0].offset(), Offset::Offset(1)); + assert_eq!(res_tpl.elements()[0].error(), Ok(())); + + // Ensure that the low water mark has advanced to 1. + let (lo, hi) = producer_client + .client() + .fetch_watermarks(&topic, 0, timeout) + .unwrap(); + assert_eq!(lo, 1); + assert_eq!(hi, 5); + + // Delete the record at offset 1 and also include an invalid partition in + // the request. The invalid partition should not cause the request to fail, + // but we should be able to see the per-partition error in the returned + // topic partition list. + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&topic, 0, Offset::Offset(2)) + .unwrap(); + tpl.add_partition_offset(&topic, 1, Offset::Offset(1)) + .unwrap(); + let res_tpl = admin_client.delete_records(&tpl, &opts).await.unwrap(); + assert_eq!(res_tpl.count(), 2); + assert_eq!(res_tpl.elements()[0].topic(), topic); + assert_eq!(res_tpl.elements()[0].partition(), 0); + assert_eq!(res_tpl.elements()[0].offset(), Offset::Offset(2)); + assert_eq!(res_tpl.elements()[0].error(), Ok(())); + assert_eq!(res_tpl.elements()[1].topic(), topic); + assert_eq!(res_tpl.elements()[1].partition(), 1); + assert_eq!( + res_tpl.elements()[1].error(), + Err(KafkaError::OffsetFetch(RDKafkaErrorCode::UnknownPartition)) + ); + + // Ensure that the low water mark has advanced to 2. + let (lo, hi) = producer_client + .client() + .fetch_watermarks(&topic, 0, timeout) + .unwrap(); + assert_eq!(lo, 2); + assert_eq!(hi, 5); + + // Delete all records up to offset 5. + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&topic, 0, Offset::End).unwrap(); + let res_tpl = admin_client.delete_records(&tpl, &opts).await.unwrap(); + assert_eq!(res_tpl.count(), 1); + assert_eq!(res_tpl.elements()[0].topic(), topic); + assert_eq!(res_tpl.elements()[0].partition(), 0); + assert_eq!(res_tpl.elements()[0].offset(), Offset::Offset(5)); + assert_eq!(res_tpl.elements()[0].error(), Ok(())); + + // Ensure that the low water mark has advanced to 5. + let (lo, hi) = producer_client + .client() + .fetch_watermarks(&topic, 0, timeout) + .unwrap(); + assert_eq!(lo, 5); + assert_eq!(hi, 5); +} diff --git a/tests/utils/producer.rs b/tests/utils/producer/base_producer.rs similarity index 100% rename from tests/utils/producer.rs rename to tests/utils/producer/base_producer.rs diff --git a/tests/utils/producer/future_producer.rs b/tests/utils/producer/future_producer.rs new file mode 100644 index 000000000..8d4891ad7 --- /dev/null +++ b/tests/utils/producer/future_producer.rs @@ -0,0 +1,12 @@ +use anyhow::Context; +use rdkafka::config::FromClientConfig; +use rdkafka::producer::FutureProducer; +use rdkafka::ClientConfig; + +pub async fn create_producer(bootstrap_servers: &str) -> anyhow::Result { + let mut producer_client_config = ClientConfig::default(); + producer_client_config.set("bootstrap.servers", bootstrap_servers); + let future_producer = FutureProducer::from_config(&producer_client_config) + .context("couldn't create producer client")?; + Ok(future_producer) +} diff --git a/tests/utils/producer/mod.rs b/tests/utils/producer/mod.rs new file mode 100644 index 000000000..f09b39c97 --- /dev/null +++ b/tests/utils/producer/mod.rs @@ -0,0 +1,2 @@ +pub mod base_producer; +pub mod future_producer; From 50528fba437791a4e2b842176b3d9cfe06c3ba81 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Sat, 11 Oct 2025 22:04:40 -0400 Subject: [PATCH 16/56] fix: test_configs --- tests/admin.rs | 86 ++++++++++++++++++++- tests/producer.rs | 5 +- tests/test_admin.rs | 180 -------------------------------------------- 3 files changed, 87 insertions(+), 184 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index 8fe0307b5..616614065 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -5,8 +5,8 @@ use crate::utils::rand::rand_test_topic; use crate::utils::{get_broker_version, KafkaVersion}; use backon::{BlockingRetryable, ExponentialBuilder}; use rdkafka::admin::{ - AdminOptions, ConfigEntry, ConfigSource, NewPartitions, NewTopic, ResourceSpecifier, - TopicReplication, + AdminOptions, AlterConfig, ConfigEntry, ConfigSource, NewPartitions, NewTopic, + OwnedResourceSpecifier, ResourceSpecifier, TopicReplication, }; use rdkafka::error::KafkaError; use rdkafka::producer::{FutureRecord, Producer}; @@ -541,3 +541,85 @@ async fn test_delete_records() { assert_eq!(lo, 5); assert_eq!(hi, 5); } + +#[tokio::test] +async fn test_configs() { + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + let opts = AdminOptions::new(); + let broker = ResourceSpecifier::Broker(utils::BROKER_ID); + + let res = admin_client + .describe_configs(&[broker], &opts) + .await + .expect("describe configs failed"); + let config = &res[0].as_ref().expect("describe configs failed"); + let orig_val = config + .get("log.flush.interval.messages") + .expect("original config entry missing") + .value + .as_ref() + .expect("original value missing"); + + let config = AlterConfig::new(broker).set("log.flush.interval.messages", "1234"); + let res = admin_client + .alter_configs(&[config], &opts) + .await + .expect("alter configs failed"); + assert_eq!(res, &[Ok(OwnedResourceSpecifier::Broker(utils::BROKER_ID))]); + + let mut tries = 0; + loop { + let res = admin_client + .describe_configs(&[broker], &opts) + .await + .expect("describe configs failed"); + let config = &res[0].as_ref().expect("describe configs failed"); + let entry = config.get("log.flush.interval.messages"); + let expected_entry = if get_broker_version(&kafka_context) < KafkaVersion(1, 1, 0, 0) { + // Pre-1.1, the AlterConfig operation will silently fail, and the + // config will remain unchanged, which I guess is worth testing. + ConfigEntry { + name: "log.flush.interval.messages".into(), + value: Some(orig_val.clone()), + source: ConfigSource::Default, + is_read_only: true, + is_default: true, + is_sensitive: false, + } + } else { + ConfigEntry { + name: "log.flush.interval.messages".into(), + value: Some("1234".into()), + source: ConfigSource::DynamicBroker, + is_read_only: false, + is_default: false, + is_sensitive: false, + } + }; + if entry == Some(&expected_entry) { + break; + } else if tries >= 5 { + panic!("{:?} != {:?}", entry, Some(&expected_entry)); + } else { + tries += 1; + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + + let config = AlterConfig::new(broker).set("log.flush.interval.ms", orig_val); + let res = admin_client + .alter_configs(&[config], &opts) + .await + .expect("alter configs failed"); + assert_eq!(res, &[Ok(OwnedResourceSpecifier::Broker(utils::BROKER_ID))]); +} diff --git a/tests/producer.rs b/tests/producer.rs index a6bb1580e..0224606d5 100644 --- a/tests/producer.rs +++ b/tests/producer.rs @@ -2,7 +2,7 @@ use crate::utils::admin::{create_admin_client, create_topic}; use crate::utils::consumer::{create_subscribed_base_consumer, poll_x_times_for_messages}; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; -use crate::utils::producer::{create_producer, send_record}; +use crate::utils::producer::base_producer::create_producer; use crate::utils::rand::rand_test_topic; use rdkafka::producer::BaseRecord; use rdkafka::Message; @@ -58,7 +58,8 @@ pub async fn test_basic_produce() { let record = BaseRecord::to(&test_topic_name) // destination topic .key(&[1, 2, 3, 4]) // message key .payload("content"); // message payload - let send_record_result = send_record(&base_producer, record).await; + let send_record_result = + crate::utils::producer::base_producer::send_record(&base_producer, record).await; if send_record_result.is_err() { panic!("could not send record: {}", send_record_result.unwrap_err()); } diff --git a/tests/test_admin.rs b/tests/test_admin.rs index 5d5a90975..1d228f282 100644 --- a/tests/test_admin.rs +++ b/tests/test_admin.rs @@ -119,186 +119,6 @@ fn verify_delete(topic: &str) { .unwrap() } -/// Test the admin client's delete records functionality. -#[tokio::test] -async fn test_delete_records() { - let producer = create_config().create::>().unwrap(); - let admin_client = create_admin_client(); - let timeout = Some(Duration::from_secs(1)); - let opts = AdminOptions::new().operation_timeout(timeout); - let topic = rand_test_topic("test_delete_records"); - let make_record = || FutureRecord::::to(&topic).payload("data"); - - // Create a topic with a single partition. - admin_client - .create_topics( - &[NewTopic::new(&topic, 1, TopicReplication::Fixed(1))], - &opts, - ) - .await - .expect("topic creation failed"); - - // Ensure that the topic begins with low and high water marks of 0. - let (lo, hi) = (|| producer.client().fetch_watermarks(&topic, 0, timeout)) - .retry(ExponentialBuilder::default().with_max_delay(Duration::from_secs(5))) - .call() - .unwrap(); - assert_eq!(lo, 0); - assert_eq!(hi, 0); - - // Produce five messages to the topic. - for _ in 0..5 { - producer.send(make_record(), timeout).await.unwrap(); - } - - // Ensure that the high water mark has advanced to 5. - let (lo, hi) = producer - .client() - .fetch_watermarks(&topic, 0, timeout) - .unwrap(); - assert_eq!(lo, 0); - assert_eq!(hi, 5); - - // Delete the record at offset 0. - let mut tpl = TopicPartitionList::new(); - tpl.add_partition_offset(&topic, 0, Offset::Offset(1)) - .unwrap(); - let res_tpl = admin_client.delete_records(&tpl, &opts).await.unwrap(); - assert_eq!(res_tpl.count(), 1); - assert_eq!(res_tpl.elements()[0].topic(), topic); - assert_eq!(res_tpl.elements()[0].partition(), 0); - assert_eq!(res_tpl.elements()[0].offset(), Offset::Offset(1)); - assert_eq!(res_tpl.elements()[0].error(), Ok(())); - - // Ensure that the low water mark has advanced to 1. - let (lo, hi) = producer - .client() - .fetch_watermarks(&topic, 0, timeout) - .unwrap(); - assert_eq!(lo, 1); - assert_eq!(hi, 5); - - // Delete the record at offset 1 and also include an invalid partition in - // the request. The invalid partition should not cause the request to fail, - // but we should be able to see the per-partition error in the returned - // topic partition list. - let mut tpl = TopicPartitionList::new(); - tpl.add_partition_offset(&topic, 0, Offset::Offset(2)) - .unwrap(); - tpl.add_partition_offset(&topic, 1, Offset::Offset(1)) - .unwrap(); - let res_tpl = admin_client.delete_records(&tpl, &opts).await.unwrap(); - assert_eq!(res_tpl.count(), 2); - assert_eq!(res_tpl.elements()[0].topic(), topic); - assert_eq!(res_tpl.elements()[0].partition(), 0); - assert_eq!(res_tpl.elements()[0].offset(), Offset::Offset(2)); - assert_eq!(res_tpl.elements()[0].error(), Ok(())); - assert_eq!(res_tpl.elements()[1].topic(), topic); - assert_eq!(res_tpl.elements()[1].partition(), 1); - assert_eq!( - res_tpl.elements()[1].error(), - Err(KafkaError::OffsetFetch(RDKafkaErrorCode::UnknownPartition)) - ); - - // Ensure that the low water mark has advanced to 2. - let (lo, hi) = producer - .client() - .fetch_watermarks(&topic, 0, timeout) - .unwrap(); - assert_eq!(lo, 2); - assert_eq!(hi, 5); - - // Delete all records up to offset 5. - let mut tpl = TopicPartitionList::new(); - tpl.add_partition_offset(&topic, 0, Offset::End).unwrap(); - let res_tpl = admin_client.delete_records(&tpl, &opts).await.unwrap(); - assert_eq!(res_tpl.count(), 1); - assert_eq!(res_tpl.elements()[0].topic(), topic); - assert_eq!(res_tpl.elements()[0].partition(), 0); - assert_eq!(res_tpl.elements()[0].offset(), Offset::Offset(5)); - assert_eq!(res_tpl.elements()[0].error(), Ok(())); - - // Ensure that the low water mark has advanced to 5. - let (lo, hi) = producer - .client() - .fetch_watermarks(&topic, 0, timeout) - .unwrap(); - assert_eq!(lo, 5); - assert_eq!(hi, 5); -} - -#[tokio::test] -async fn test_configs() { - let admin_client = create_admin_client(); - let opts = AdminOptions::new(); - let broker = ResourceSpecifier::Broker(0); - - let res = admin_client - .describe_configs(&[broker], &opts) - .await - .expect("describe configs failed"); - let config = &res[0].as_ref().expect("describe configs failed"); - let orig_val = config - .get("log.flush.interval.messages") - .expect("original config entry missing") - .value - .as_ref() - .expect("original value missing"); - - let config = AlterConfig::new(broker).set("log.flush.interval.messages", "1234"); - let res = admin_client - .alter_configs(&[config], &opts) - .await - .expect("alter configs failed"); - assert_eq!(res, &[Ok(OwnedResourceSpecifier::Broker(0))]); - - let mut tries = 0; - loop { - let res = admin_client - .describe_configs(&[broker], &opts) - .await - .expect("describe configs failed"); - let config = &res[0].as_ref().expect("describe configs failed"); - let entry = config.get("log.flush.interval.messages"); - let expected_entry = if get_broker_version() < KafkaVersion(1, 1, 0, 0) { - // Pre-1.1, the AlterConfig operation will silently fail, and the - // config will remain unchanged, which I guess is worth testing. - ConfigEntry { - name: "log.flush.interval.messages".into(), - value: Some(orig_val.clone()), - source: ConfigSource::Default, - is_read_only: true, - is_default: true, - is_sensitive: false, - } - } else { - ConfigEntry { - name: "log.flush.interval.messages".into(), - value: Some("1234".into()), - source: ConfigSource::DynamicBroker, - is_read_only: false, - is_default: false, - is_sensitive: false, - } - }; - if entry == Some(&expected_entry) { - break; - } else if tries >= 5 { - panic!("{:?} != {:?}", entry, Some(&expected_entry)); - } else { - tries += 1; - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - - let config = AlterConfig::new(broker).set("log.flush.interval.ms", orig_val); - let res = admin_client - .alter_configs(&[config], &opts) - .await - .expect("alter configs failed"); - assert_eq!(res, &[Ok(OwnedResourceSpecifier::Broker(0))]); -} - #[tokio::test] async fn test_groups() { let admin_client = create_admin_client(); From bc126040883f5d77ecf306929f0a86d12600a079 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Sun, 12 Oct 2025 19:06:11 -0400 Subject: [PATCH 17/56] fix: test_delete_records --- tests/consumer_groups.rs | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/consumer_groups.rs diff --git a/tests/consumer_groups.rs b/tests/consumer_groups.rs new file mode 100644 index 000000000..e69de29bb From 89ba998a2fd739c391bf9d2aa75d2d084eba5dff Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Mon, 13 Oct 2025 10:47:06 -0400 Subject: [PATCH 18/56] fix: test_consumer_groups_deletion and test_delete_unknown_group --- tests/admin.rs | 50 ++++++++++++++++++++++--- tests/consumer_groups.rs | 79 ++++++++++++++++++++++++++++++++++++++++ tests/producer.rs | 3 +- tests/test_admin.rs | 50 ------------------------- tests/utils/consumer.rs | 39 ++++++++++++++++++-- 5 files changed, 161 insertions(+), 60 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index 616614065..79949a742 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -1,11 +1,11 @@ use crate::utils::admin::create_topic; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; -use crate::utils::rand::rand_test_topic; +use crate::utils::rand::{rand_test_group, rand_test_topic}; use crate::utils::{get_broker_version, KafkaVersion}; use backon::{BlockingRetryable, ExponentialBuilder}; use rdkafka::admin::{ - AdminOptions, AlterConfig, ConfigEntry, ConfigSource, NewPartitions, NewTopic, + AdminOptions, AlterConfig, ConfigEntry, ConfigSource, GroupResult, NewPartitions, NewTopic, OwnedResourceSpecifier, ResourceSpecifier, TopicReplication, }; use rdkafka::error::KafkaError; @@ -69,7 +69,7 @@ pub async fn test_topic_create_and_delete() { // Create consumer client let consumer_client = - utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers) + utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers, None) .await .expect("could not create consumer client"); @@ -276,7 +276,7 @@ pub async fn test_incorrect_replication_factors_are_ignored_when_creating_partit // Create consumer client let consumer_client = - utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers) + utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers, None) .await .expect("could not create consumer client"); @@ -361,7 +361,7 @@ pub async fn test_mixed_success_results() { // Create consumer client let consumer_client = - utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers) + utils::consumer::create_unsubscribed_base_consumer(&kafka_context.bootstrap_servers, None) .await .expect("could not create consumer client"); @@ -623,3 +623,43 @@ async fn test_configs() { .expect("alter configs failed"); assert_eq!(res, &[Ok(OwnedResourceSpecifier::Broker(utils::BROKER_ID))]); } + +#[tokio::test] +async fn test_groups() { + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + let opts = AdminOptions::new(); + + // Verify that deleting a valid and invalid group results in a mixed result + // set. + { + let group_name = rand_test_group(); + let unknown_group_name = rand_test_group(); + create_consumer_group(&group_name).await; + let res = admin_client + .delete_groups( + &[&group_name, &unknown_group_name], + &AdminOptions::default(), + ) + .await; + assert_eq!( + res, + Ok(vec![ + Ok(group_name.to_string()), + Err(( + unknown_group_name.to_string(), + RDKafkaErrorCode::GroupIdNotFound + )) + ]) + ); + } +} diff --git a/tests/consumer_groups.rs b/tests/consumer_groups.rs index e69de29bb..006ac9b0f 100644 --- a/tests/consumer_groups.rs +++ b/tests/consumer_groups.rs @@ -0,0 +1,79 @@ +use crate::utils::containers::KafkaContext; +use crate::utils::logging::init_test_logger; +use crate::utils::rand::{rand_test_group, rand_test_topic}; +use rdkafka::admin::{AdminOptions, GroupResult, NewTopic, TopicReplication}; +use rdkafka_sys::RDKafkaErrorCode; + +mod utils; + +/// Verify that a valid group can be deleted. +#[tokio::test] +pub async fn test_consumer_groups_deletion() { + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + + // Create consumer_client + let group_name = rand_test_group(); + let topic_name = rand_test_topic("test_topic"); + let consumer_client = utils::consumer::create_unsubscribed_base_consumer( + &kafka_context.bootstrap_servers, + Some(&group_name), + ) + .await + .expect("could not create subscribed base consumer"); + + admin_client + .create_topics( + &[NewTopic { + name: &topic_name, + num_partitions: 1, + replication: TopicReplication::Fixed(1), + config: vec![], + }], + &AdminOptions::default(), + ) + .await + .expect("topic creation failed"); + + utils::consumer::create_consumer_group_on_topic(&consumer_client, &topic_name) + .await + .expect("could not create group"); + let res = admin_client + .delete_groups(&[&group_name], &AdminOptions::default()) + .await + .expect("could not delete groups"); + assert_eq!(res, [Ok(group_name.to_string())]); +} + +/// Verify that attempting to delete an unknown group returns a "group not +/// found" error. +#[tokio::test] +pub async fn delete_unknown_group() { + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + + let unknown_group_name = rand_test_group(); + let res = admin_client + .delete_groups(&[&unknown_group_name], &AdminOptions::default()) + .await; + let expected: GroupResult = Err((unknown_group_name, RDKafkaErrorCode::NotCoordinator)); + assert_eq!(res, Ok(vec![expected])); +} diff --git a/tests/producer.rs b/tests/producer.rs index 0224606d5..b41a9d394 100644 --- a/tests/producer.rs +++ b/tests/producer.rs @@ -39,7 +39,8 @@ pub async fn test_basic_produce() { } let consumer_result = - create_subscribed_base_consumer(&kafka_context.bootstrap_servers, &test_topic_name).await; + create_subscribed_base_consumer(&kafka_context.bootstrap_servers, None, &test_topic_name) + .await; let Ok(consumer) = consumer_result else { panic!( "could not create consumer: {}", diff --git a/tests/test_admin.rs b/tests/test_admin.rs index 1d228f282..7ba9f7d8d 100644 --- a/tests/test_admin.rs +++ b/tests/test_admin.rs @@ -119,56 +119,6 @@ fn verify_delete(topic: &str) { .unwrap() } -#[tokio::test] -async fn test_groups() { - let admin_client = create_admin_client(); - - // Verify that a valid group can be deleted. - { - let group_name = rand_test_group(); - create_consumer_group(&group_name).await; - let res = admin_client - .delete_groups(&[&group_name], &AdminOptions::default()) - .await; - assert_eq!(res, Ok(vec![Ok(group_name.to_string())])); - } - - // Verify that attempting to delete an unknown group returns a "group not - // found" error. - { - let unknown_group_name = rand_test_group(); - let res = admin_client - .delete_groups(&[&unknown_group_name], &AdminOptions::default()) - .await; - let expected: GroupResult = Err((unknown_group_name, RDKafkaErrorCode::GroupIdNotFound)); - assert_eq!(res, Ok(vec![expected])); - } - - // Verify that deleting a valid and invalid group results in a mixed result - // set. - { - let group_name = rand_test_group(); - let unknown_group_name = rand_test_group(); - create_consumer_group(&group_name).await; - let res = admin_client - .delete_groups( - &[&group_name, &unknown_group_name], - &AdminOptions::default(), - ) - .await; - assert_eq!( - res, - Ok(vec![ - Ok(group_name.to_string()), - Err(( - unknown_group_name.to_string(), - RDKafkaErrorCode::GroupIdNotFound - )) - ]) - ); - } -} - // Tests whether each admin operation properly reports an error if the entire // request fails. The original implementations failed to check this, resulting // in confusing situations where a failed admin request would return Ok([]). diff --git a/tests/utils/consumer.rs b/tests/utils/consumer.rs index 954c69417..4fd364046 100644 --- a/tests/utils/consumer.rs +++ b/tests/utils/consumer.rs @@ -2,17 +2,19 @@ use crate::utils::rand::rand_test_group; use anyhow::{bail, Context}; use backon::{BlockingRetryable, ExponentialBuilder}; use rdkafka::config::FromClientConfig; -use rdkafka::consumer::{BaseConsumer, Consumer}; +use rdkafka::consumer::{BaseConsumer, CommitMode, Consumer}; use rdkafka::message::BorrowedMessage; use rdkafka::metadata::Metadata; -use rdkafka::ClientConfig; +use rdkafka::{ClientConfig, TopicPartitionList}; use std::time::Duration; pub async fn create_subscribed_base_consumer( bootstrap_servers: &str, + consumer_group_option: Option<&str>, test_topic: &str, ) -> anyhow::Result { - let unsubscribed_base_consumer = create_unsubscribed_base_consumer(bootstrap_servers).await?; + let unsubscribed_base_consumer = + create_unsubscribed_base_consumer(bootstrap_servers, consumer_group_option).await?; unsubscribed_base_consumer .subscribe(&[test_topic]) .context("Failed to subscribe to topic")?; @@ -21,9 +23,14 @@ pub async fn create_subscribed_base_consumer( pub async fn create_unsubscribed_base_consumer( bootstrap_servers: &str, + consumer_group_option: Option<&str>, ) -> anyhow::Result { + let consumer_group_name = match consumer_group_option { + Some(consumer_group_name) => consumer_group_name, + None => &rand_test_group(), + }; let mut consumer_client_config = ClientConfig::default(); - consumer_client_config.set("group.id", rand_test_group()); + consumer_client_config.set("group.id", consumer_group_name); consumer_client_config.set("client.id", "rdkafka_integration_test_client"); consumer_client_config.set("bootstrap.servers", bootstrap_servers); consumer_client_config.set("enable.partition.eof", "false"); @@ -97,3 +104,27 @@ pub fn verify_topic_deleted(consumer: &BaseConsumer, topic: &str) -> anyhow::Res .retry(ExponentialBuilder::default().with_max_delay(Duration::from_secs(5))) .call() } + +pub async fn create_consumer_group_on_topic( + consumer_client: &BaseConsumer, + topic_name: &str, +) -> anyhow::Result<()> { + let topic_partition_list = { + let mut lst = TopicPartitionList::new(); + lst.add_partition(topic_name, 0); + lst + }; + consumer_client + .assign(&topic_partition_list) + .context("assign topic partition list failed")?; + consumer_client + .fetch_metadata(None, Duration::from_secs(3)) + .context("unable to fetch metadata")?; + (|| consumer_client.store_offset(topic_name, 0, -1)) + .retry(ExponentialBuilder::default().with_max_delay(Duration::from_secs(5))) + .call() + .context("store offset failed")?; + consumer_client + .commit_consumer_state(CommitMode::Sync) + .context("commit the consumer state failed") +} From 74c1ec02f4ddc813b177a2c1abcda62340fd7f86 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Mon, 13 Oct 2025 10:51:41 -0400 Subject: [PATCH 19/56] fix: test_consumer_group_action_mixed_results --- tests/admin.rs | 24 --------------- tests/consumer_groups.rs | 64 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index 79949a742..0f26d25be 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -638,28 +638,4 @@ async fn test_groups() { .await .expect("could not create admin client"); let opts = AdminOptions::new(); - - // Verify that deleting a valid and invalid group results in a mixed result - // set. - { - let group_name = rand_test_group(); - let unknown_group_name = rand_test_group(); - create_consumer_group(&group_name).await; - let res = admin_client - .delete_groups( - &[&group_name, &unknown_group_name], - &AdminOptions::default(), - ) - .await; - assert_eq!( - res, - Ok(vec![ - Ok(group_name.to_string()), - Err(( - unknown_group_name.to_string(), - RDKafkaErrorCode::GroupIdNotFound - )) - ]) - ); - } } diff --git a/tests/consumer_groups.rs b/tests/consumer_groups.rs index 006ac9b0f..71d44c8fd 100644 --- a/tests/consumer_groups.rs +++ b/tests/consumer_groups.rs @@ -1,3 +1,4 @@ +use crate::utils::consumer; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; use crate::utils::rand::{rand_test_group, rand_test_topic}; @@ -57,7 +58,7 @@ pub async fn test_consumer_groups_deletion() { /// Verify that attempting to delete an unknown group returns a "group not /// found" error. #[tokio::test] -pub async fn delete_unknown_group() { +pub async fn test_delete_unknown_group() { init_test_logger(); // Get Kafka container context. @@ -77,3 +78,64 @@ pub async fn delete_unknown_group() { let expected: GroupResult = Err((unknown_group_name, RDKafkaErrorCode::NotCoordinator)); assert_eq!(res, Ok(vec![expected])); } + +/// Verify that deleting a valid and invalid group results in a mixed result +/// set. +#[tokio::test] +pub async fn test_consumer_group_action_mixed_results() { + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // Create admin client + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + + // Create consumer_client + let group_name = rand_test_group(); + let topic_name = rand_test_topic("test_topic"); + let consumer_client = utils::consumer::create_unsubscribed_base_consumer( + &kafka_context.bootstrap_servers, + Some(&group_name), + ) + .await + .expect("could not create subscribed base consumer"); + + admin_client + .create_topics( + &[NewTopic { + name: &topic_name, + num_partitions: 1, + replication: TopicReplication::Fixed(1), + config: vec![], + }], + &AdminOptions::default(), + ) + .await + .expect("topic creation failed"); + + let unknown_group_name = rand_test_group(); + consumer::create_consumer_group_on_topic(&consumer_client, &topic_name) + .await + .expect("could not create group"); + let res = admin_client + .delete_groups( + &[&group_name, &unknown_group_name], + &AdminOptions::default(), + ) + .await; + assert_eq!( + res, + Ok(vec![ + Ok(group_name.to_string()), + Err(( + unknown_group_name.to_string(), + RDKafkaErrorCode::GroupIdNotFound + )) + ]) + ); +} From 8c9ab9bf32add66c4827bacb132842ef22f57f3d Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Mon, 13 Oct 2025 11:01:56 -0400 Subject: [PATCH 20/56] fix: remove test_admin.rs --- tests/admin.rs | 62 ++++++++++---- tests/test_admin.rs | 166 -------------------------------------- tests/utils/containers.rs | 2 +- 3 files changed, 46 insertions(+), 184 deletions(-) delete mode 100644 tests/test_admin.rs diff --git a/tests/admin.rs b/tests/admin.rs index 0f26d25be..7010b3ddd 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -4,15 +4,13 @@ use crate::utils::logging::init_test_logger; use crate::utils::rand::{rand_test_group, rand_test_topic}; use crate::utils::{get_broker_version, KafkaVersion}; use backon::{BlockingRetryable, ExponentialBuilder}; -use rdkafka::admin::{ - AdminOptions, AlterConfig, ConfigEntry, ConfigSource, GroupResult, NewPartitions, NewTopic, - OwnedResourceSpecifier, ResourceSpecifier, TopicReplication, -}; +use rdkafka::admin::{AdminClient, AdminOptions, AlterConfig, ConfigEntry, ConfigSource, GroupResult, NewPartitions, NewTopic, OwnedResourceSpecifier, ResourceSpecifier, TopicReplication}; use rdkafka::error::KafkaError; use rdkafka::producer::{FutureRecord, Producer}; -use rdkafka::{Offset, TopicPartitionList}; +use rdkafka::{ClientConfig, Offset, TopicPartitionList}; use rdkafka_sys::RDKafkaErrorCode; use std::time::Duration; +use rdkafka::client::DefaultClientContext; #[path = "utils/mod.rs"] mod utils; @@ -624,18 +622,48 @@ async fn test_configs() { assert_eq!(res, &[Ok(OwnedResourceSpecifier::Broker(utils::BROKER_ID))]); } +/// Tests whether each admin operation properly reports an error if the entire +/// request fails. The original implementations failed to check this, resulting +/// in confusing situations where a failed admin request would return Ok([]). #[tokio::test] -async fn test_groups() { - init_test_logger(); +async fn test_event_errors() { + // Configure an admin client to target a Kafka server that doesn't exist, + // then set an impossible timeout. This will ensure that every request fails + // with an OperationTimedOut error, assuming, of course, that the request + // passes client-side validation. + let admin_client = ClientConfig::new() + .set("bootstrap.servers", "noexist") + .create::>() + .expect("admin client creation failed"); + let opts = AdminOptions::new().request_timeout(Some(Duration::from_nanos(1))); + + let res = admin_client.create_topics(&[], &opts).await; + assert_eq!( + res, + Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) + ); - // Get Kafka container context. - let kafka_context = KafkaContext::shared() - .await - .expect("could not create kafka context"); + let res = admin_client.create_partitions(&[], &opts).await; + assert_eq!( + res, + Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) + ); - // Create admin client - let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) - .await - .expect("could not create admin client"); - let opts = AdminOptions::new(); -} + let res = admin_client.delete_topics(&[], &opts).await; + assert_eq!( + res, + Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) + ); + + let res = admin_client.describe_configs(&[], &opts).await; + assert_eq!( + res.err(), + Some(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) + ); + + let res = admin_client.alter_configs(&[], &opts).await; + assert_eq!( + res, + Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) + ); +} \ No newline at end of file diff --git a/tests/test_admin.rs b/tests/test_admin.rs deleted file mode 100644 index 7ba9f7d8d..000000000 --- a/tests/test_admin.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! Test administrative commands using the admin API. - -use std::time::Duration; - -use backon::{BlockingRetryable, ExponentialBuilder}; - -use rdkafka::admin::{ - AdminClient, AdminOptions, AlterConfig, ConfigEntry, ConfigSource, GroupResult, NewPartitions, - NewTopic, OwnedResourceSpecifier, ResourceSpecifier, TopicReplication, -}; -use rdkafka::client::DefaultClientContext; -use rdkafka::consumer::{BaseConsumer, CommitMode, Consumer, DefaultConsumerContext}; -use rdkafka::error::{KafkaError, RDKafkaErrorCode}; -use rdkafka::metadata::Metadata; -use rdkafka::producer::{FutureProducer, FutureRecord, Producer}; -use rdkafka::{ClientConfig, Offset, TopicPartitionList}; - -use crate::utils::logging::init_test_logger; -use crate::utils::rand::{rand_test_group, rand_test_topic}; -use crate::utils::*; - -mod utils; - -fn create_config() -> ClientConfig { - let mut config = ClientConfig::new(); - config.set("bootstrap.servers", get_bootstrap_server().as_str()); - config -} - -fn create_admin_client() -> AdminClient { - init_test_logger(); - create_config() - .create() - .expect("admin client creation failed") -} - -async fn create_consumer_group(consumer_group_name: &str) { - let admin_client = create_admin_client(); - let topic_name = &rand_test_topic(consumer_group_name); - let consumer: BaseConsumer = create_config() - .set("group.id", consumer_group_name) - .create() - .expect("create consumer failed"); - - admin_client - .create_topics( - &[NewTopic { - name: topic_name, - num_partitions: 1, - replication: TopicReplication::Fixed(1), - config: vec![], - }], - &AdminOptions::default(), - ) - .await - .expect("topic creation failed"); - let topic_partition_list = { - let mut lst = TopicPartitionList::new(); - lst.add_partition(topic_name, 0); - lst - }; - consumer - .assign(&topic_partition_list) - .expect("assign topic partition list failed"); - consumer - .fetch_metadata(None, Duration::from_secs(3)) - .expect("unable to fetch metadata"); - (|| consumer.store_offset(topic_name, 0, -1)) - .retry(ExponentialBuilder::default().with_max_delay(Duration::from_secs(5))) - .call() - .expect("store offset failed"); - consumer - .commit_consumer_state(CommitMode::Sync) - .expect("commit the consumer state failed"); -} - -fn fetch_metadata(topic: &str) -> Metadata { - let consumer: BaseConsumer = - create_config().create().expect("consumer creation failed"); - let timeout = Some(Duration::from_secs(1)); - - (|| { - let metadata = consumer - .fetch_metadata(Some(topic), timeout) - .map_err(|e| e.to_string())?; - if metadata.topics().is_empty() { - Err("metadata fetch returned no topics".to_string())? - } - let topic = &metadata.topics()[0]; - if topic.partitions().is_empty() { - Err("metadata fetch returned a topic with no partitions".to_string())? - } - Ok::<_, String>(metadata) - }) - .retry(ExponentialBuilder::default().with_max_delay(Duration::from_secs(5))) - .call() - .unwrap() -} - -fn verify_delete(topic: &str) { - let consumer: BaseConsumer = - create_config().create().expect("consumer creation failed"); - let timeout = Some(Duration::from_secs(1)); - - (|| { - // Asking about the topic specifically will recreate it (under the - // default Kafka configuration, at least) so we have to ask for the list - // of all topics and search through it. - let metadata = consumer - .fetch_metadata(None, timeout) - .map_err(|e| e.to_string())?; - if metadata.topics().iter().any(|t| t.name() == topic) { - Err(format!("topic {} still exists", topic))? - } - Ok::<(), String>(()) - }) - .retry(ExponentialBuilder::default().with_max_delay(Duration::from_secs(5))) - .call() - .unwrap() -} - -// Tests whether each admin operation properly reports an error if the entire -// request fails. The original implementations failed to check this, resulting -// in confusing situations where a failed admin request would return Ok([]). -#[tokio::test] -async fn test_event_errors() { - // Configure an admin client to target a Kafka server that doesn't exist, - // then set an impossible timeout. This will ensure that every request fails - // with an OperationTimedOut error, assuming, of course, that the request - // passes client-side validation. - let admin_client = ClientConfig::new() - .set("bootstrap.servers", "noexist") - .create::>() - .expect("admin client creation failed"); - let opts = AdminOptions::new().request_timeout(Some(Duration::from_nanos(1))); - - let res = admin_client.create_topics(&[], &opts).await; - assert_eq!( - res, - Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) - ); - - let res = admin_client.create_partitions(&[], &opts).await; - assert_eq!( - res, - Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) - ); - - let res = admin_client.delete_topics(&[], &opts).await; - assert_eq!( - res, - Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) - ); - - let res = admin_client.describe_configs(&[], &opts).await; - assert_eq!( - res.err(), - Some(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) - ); - - let res = admin_client.alter_configs(&[], &opts).await; - assert_eq!( - res, - Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) - ); -} diff --git a/tests/utils/containers.rs b/tests/utils/containers.rs index 560d7ce7e..359d187f9 100644 --- a/tests/utils/containers.rs +++ b/tests/utils/containers.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use testcontainers_modules::kafka::apache::Kafka; use testcontainers_modules::testcontainers::core::ContainerPort; use testcontainers_modules::testcontainers::runners::AsyncRunner; -use testcontainers_modules::testcontainers::{ContainerAsync, Image}; +use testcontainers_modules::testcontainers::{ContainerAsync, Image, ImageExt}; use tokio::sync::OnceCell; pub struct KafkaContext { From 568431221b0a3d175c9db9580c1d993a9695588e Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 17 Oct 2025 18:02:46 -0400 Subject: [PATCH 21/56] fix: test_produce_consume_base_assign --- tests/admin.rs | 9 +- ..._high_consumers.rs => stream_consumers.rs} | 135 +++++++++++++++--- tests/utils/admin.rs | 6 +- tests/utils/{consumer.rs => consumer/mod.rs} | 2 + tests/utils/consumer/stream_consumer.rs | 19 +++ tests/utils/containers.rs | 2 +- tests/utils/mod.rs | 1 + tests/utils/topics.rs | 44 ++++++ 8 files changed, 192 insertions(+), 26 deletions(-) rename tests/{test_high_consumers.rs => stream_consumers.rs} (84%) rename tests/utils/{consumer.rs => consumer/mod.rs} (99%) create mode 100644 tests/utils/consumer/stream_consumer.rs create mode 100644 tests/utils/topics.rs diff --git a/tests/admin.rs b/tests/admin.rs index 7010b3ddd..fde2cbf58 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -4,13 +4,16 @@ use crate::utils::logging::init_test_logger; use crate::utils::rand::{rand_test_group, rand_test_topic}; use crate::utils::{get_broker_version, KafkaVersion}; use backon::{BlockingRetryable, ExponentialBuilder}; -use rdkafka::admin::{AdminClient, AdminOptions, AlterConfig, ConfigEntry, ConfigSource, GroupResult, NewPartitions, NewTopic, OwnedResourceSpecifier, ResourceSpecifier, TopicReplication}; +use rdkafka::admin::{ + AdminClient, AdminOptions, AlterConfig, ConfigEntry, ConfigSource, GroupResult, NewPartitions, + NewTopic, OwnedResourceSpecifier, ResourceSpecifier, TopicReplication, +}; +use rdkafka::client::DefaultClientContext; use rdkafka::error::KafkaError; use rdkafka::producer::{FutureRecord, Producer}; use rdkafka::{ClientConfig, Offset, TopicPartitionList}; use rdkafka_sys::RDKafkaErrorCode; use std::time::Duration; -use rdkafka::client::DefaultClientContext; #[path = "utils/mod.rs"] mod utils; @@ -666,4 +669,4 @@ async fn test_event_errors() { res, Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) ); -} \ No newline at end of file +} diff --git a/tests/test_high_consumers.rs b/tests/stream_consumers.rs similarity index 84% rename from tests/test_high_consumers.rs rename to tests/stream_consumers.rs index 88890a447..7c552db70 100644 --- a/tests/test_high_consumers.rs +++ b/tests/stream_consumers.rs @@ -15,10 +15,14 @@ use rdkafka::error::KafkaError; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::current_time_millis; use rdkafka::{Message, Timestamp}; +use rdkafka::admin::AdminOptions; use rdkafka_sys::types::RDKafkaConfRes; +use crate::utils::containers::KafkaContext; +use crate::utils::logging::init_test_logger; use crate::utils::rand::*; use crate::utils::*; +use crate::utils::admin::new_topic_vec; mod utils; @@ -71,27 +75,52 @@ async fn test_invalid_max_poll_interval() { // All produced messages should be consumed. #[tokio::test(flavor = "multi_thread")] async fn test_produce_consume_base() { - let _r = env_logger::try_init(); + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + + let num_of_messages_to_send = 100usize; let start_time = current_time_millis(); let topic_name = rand_test_topic("test_produce_consume_base"); - let message_map = populate_topic(&topic_name, 100, &value_fn, &key_fn, None, None).await; - let consumer = create_stream_consumer(&rand_test_group(), None); - consumer.subscribe(&[topic_name.as_str()]).unwrap(); + let message_map = topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + None, + ) + .await + .expect("Could not populate topic using Future producer"); + // let message_map = populate_topic(&topic_name, 100, &value_fn, &key_fn, None, None).await; + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); + consumer + .subscribe(&[topic_name.as_str()]) + .expect("could not subscribe to kafka topic"); - let _consumer_future = consumer + consumer .stream() - .take(100) + .take(num_of_messages_to_send) .for_each(|message| { match message { Ok(m) => { let id = message_map[&(m.partition(), m.offset())]; match m.timestamp() { Timestamp::CreateTime(timestamp) => assert!(timestamp >= start_time), - _ => panic!("Expected createtime for message timestamp"), + _ => panic!("Expected create time for message timestamp"), }; - assert_eq!(m.payload_view::().unwrap().unwrap(), value_fn(id)); - assert_eq!(m.key_view::().unwrap().unwrap(), key_fn(id)); + assert_eq!(m.payload_view::().unwrap().unwrap(), id.to_string()); + assert_eq!(m.key_view::().unwrap().unwrap(), id.to_string()); assert_eq!(m.topic(), topic_name.as_str()); } Err(e) => panic!("Error receiving message: {:?}", e), @@ -107,13 +136,39 @@ async fn test_produce_consume_base() { /// waker slot. #[tokio::test(flavor = "multi_thread")] async fn test_produce_consume_base_concurrent() { - let _r = env_logger::try_init(); + init_test_logger(); - let topic_name = rand_test_topic("test_produce_consume_base_concurrent"); - populate_topic(&topic_name, 100, &value_fn, &key_fn, None, None).await; + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); - let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); - consumer.subscribe(&[topic_name.as_str()]).unwrap(); + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + + let num_of_messages_to_send = 100usize; + let topic_name = rand_test_topic("test_produce_consume_base_concurrent"); + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + None, + ) + .await + .expect("Could not populate topic using Future producer"); + // let message_map = populate_topic(&topic_name, 100, &value_fn, &key_fn, None, None).await; + let consumer = Arc::new( + consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"), + ); + consumer + .subscribe(&[topic_name.as_str()]) + .expect("could not subscribe to kafka topic"); let mk_task = || { let consumer = consumer.clone(); @@ -137,13 +192,55 @@ async fn test_produce_consume_base_concurrent() { // All produced messages should be consumed. #[tokio::test(flavor = "multi_thread")] async fn test_produce_consume_base_assign() { - let _r = env_logger::try_init(); + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_produce_consume_base_assign"); - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(1), None).await; - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(2), None).await; - let consumer = create_stream_consumer(&rand_test_group(), None); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client.create_topics(&new_topic_vec(&topic_name, Some(3)), &AdminOptions::default()).await.expect("could not create topics"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + + let num_of_messages_to_send = 10usize; + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + Some(0), + ) + .await + .expect("Could not populate topic using Future producer"); + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + Some(1), + ) + .await + .expect("Could not populate topic using Future producer"); + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + Some(2), + ) + .await + .expect("Could not populate topic using Future producer"); + + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); let mut tpl = TopicPartitionList::new(); tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) .unwrap(); diff --git a/tests/utils/admin.rs b/tests/utils/admin.rs index 70e7295cf..9de144d72 100644 --- a/tests/utils/admin.rs +++ b/tests/utils/admin.rs @@ -17,7 +17,7 @@ pub async fn create_topic( topic_name: &'_ str, ) -> anyhow::Result { let topic_results = admin_client - .create_topics(&new_topic_vec(&topic_name), &AdminOptions::default()) + .create_topics(&new_topic_vec(&topic_name, None), &AdminOptions::default()) .await .context("error creating topics")?; for topic_result in topic_results { @@ -28,7 +28,7 @@ pub async fn create_topic( Ok(topic_name.to_string()) } -pub fn new_topic_vec(topic_name: &'_ str) -> Vec> { - let new_topic = NewTopic::new(&topic_name, 1, TopicReplication::Fixed(1)); +pub fn new_topic_vec(topic_name: &'_ str, num_partitions: Option) -> Vec> { + let new_topic = NewTopic::new(&topic_name, num_partitions.unwrap_or(1), TopicReplication::Fixed(1)); vec![new_topic] } diff --git a/tests/utils/consumer.rs b/tests/utils/consumer/mod.rs similarity index 99% rename from tests/utils/consumer.rs rename to tests/utils/consumer/mod.rs index 4fd364046..fa9e3e518 100644 --- a/tests/utils/consumer.rs +++ b/tests/utils/consumer/mod.rs @@ -1,3 +1,5 @@ +pub mod stream_consumer; + use crate::utils::rand::rand_test_group; use anyhow::{bail, Context}; use backon::{BlockingRetryable, ExponentialBuilder}; diff --git a/tests/utils/consumer/stream_consumer.rs b/tests/utils/consumer/stream_consumer.rs new file mode 100644 index 000000000..51201873f --- /dev/null +++ b/tests/utils/consumer/stream_consumer.rs @@ -0,0 +1,19 @@ +use anyhow::Context; +use rdkafka::ClientConfig; +use rdkafka::config::FromClientConfig; +use rdkafka::consumer::StreamConsumer; + +pub async fn create_stream_consumer( + bootstrap_server: &str, + consumer_group_option: Option<&str>, +) -> anyhow::Result { + let mut client_config = ClientConfig::default(); + client_config.set("bootstrap.servers", bootstrap_server); + client_config.set("auto.offset.reset", "earliest"); + if let Some(group) = consumer_group_option { + client_config.set("group.id", group); + } + + let stream_consumer = StreamConsumer::from_config(&client_config).context("failed to create stream consumer")?; + Ok(stream_consumer) +} diff --git a/tests/utils/containers.rs b/tests/utils/containers.rs index 359d187f9..560d7ce7e 100644 --- a/tests/utils/containers.rs +++ b/tests/utils/containers.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use testcontainers_modules::kafka::apache::Kafka; use testcontainers_modules::testcontainers::core::ContainerPort; use testcontainers_modules::testcontainers::runners::AsyncRunner; -use testcontainers_modules::testcontainers::{ContainerAsync, Image, ImageExt}; +use testcontainers_modules::testcontainers::{ContainerAsync, Image}; use tokio::sync::OnceCell; pub struct KafkaContext { diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index 27981aee2..ca9f496f2 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -6,6 +6,7 @@ pub mod containers; pub mod logging; pub mod producer; pub mod rand; +pub mod topics; use std::collections::HashMap; use std::env::{self}; diff --git a/tests/utils/topics.rs b/tests/utils/topics.rs new file mode 100644 index 000000000..d6664219a --- /dev/null +++ b/tests/utils/topics.rs @@ -0,0 +1,44 @@ +use rdkafka::producer::{FutureProducer, FutureRecord}; +use std::collections::HashMap; +use std::time::Duration; + +pub type PartitionOffset = (i32, i64); +pub type MessageId = usize; + +pub async fn populate_topic_using_future_producer( + producer: &FutureProducer, + topic_name: &str, + num_messages: usize, + partition: Option, +) -> anyhow::Result> { + let message_send_futures = (0..num_messages) + .map(|id| { + let future = async move { + producer + .send( + FutureRecord { + topic: topic_name, + payload: Some(&id.to_string()), + key: Some(&id.to_string()), + partition, + timestamp: None, + headers: None, + }, + Duration::from_secs(1), + ) + .await + }; + (id, future) + }) + .collect::>(); + + let mut message_map = HashMap::::new(); + for (id, future) in message_send_futures { + match future.await { + Ok(delivered) => message_map.insert((delivered.partition, delivered.offset), id), + Err((kafka_error, _message)) => panic!("Delivery failed: {}", kafka_error), + }; + } + + Ok(message_map) +} From 6bc6c75ce8b58817aedd0eb9561946103ba77048 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 17 Oct 2025 18:08:40 -0400 Subject: [PATCH 22/56] fix: test_produce_consume_base_unassign --- tests/stream_consumers.rs | 913 ++++++++++++++++++++------------------ 1 file changed, 478 insertions(+), 435 deletions(-) diff --git a/tests/stream_consumers.rs b/tests/stream_consumers.rs index 7c552db70..b1839e17b 100644 --- a/tests/stream_consumers.rs +++ b/tests/stream_consumers.rs @@ -269,13 +269,56 @@ async fn test_produce_consume_base_assign() { #[tokio::test(flavor = "multi_thread")] async fn test_produce_consume_base_unassign() { - let _r = env_logger::try_init(); + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + let topic_name = rand_test_topic("test_produce_consume_base_assign"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client.create_topics(&new_topic_vec(&topic_name, Some(3)), &AdminOptions::default()).await.expect("could not create topics"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); + + let num_of_messages_to_send = 10usize; + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + Some(0), + ) + .await + .expect("Could not populate topic using Future producer"); + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + Some(1), + ) + .await + .expect("Could not populate topic using Future producer"); + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + Some(2), + ) + .await + .expect("Could not populate topic using Future producer"); - let topic_name = rand_test_topic("test_produce_consume_base_unassign"); - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(1), None).await; - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(2), None).await; - let consumer = create_stream_consumer(&rand_test_group(), None); let mut tpl = TopicPartitionList::new(); tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) .unwrap(); @@ -292,432 +335,432 @@ async fn test_produce_consume_base_unassign() { assert_eq!(assignments.count(), 0); } -#[tokio::test(flavor = "multi_thread")] -async fn test_produce_consume_base_incremental_assign_and_unassign() { - let _r = env_logger::try_init(); - - let topic_name = rand_test_topic("test_produce_consume_base_incremental_assign_and_unassign"); - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(1), None).await; - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(2), None).await; - let consumer = create_stream_consumer(&rand_test_group(), None); - - // Adding a simple partition - let mut tpl = TopicPartitionList::new(); - tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) - .unwrap(); - consumer.incremental_assign(&tpl).unwrap(); - let mut assignments = consumer.assignment().unwrap(); - assert_eq!(assignments.count(), 1); - - // Adding another partition - let mut tpl = TopicPartitionList::new(); - tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) - .unwrap(); - consumer.incremental_assign(&tpl).unwrap(); - assignments = consumer.assignment().unwrap(); - assert_eq!(assignments.count(), 2); - - // Removing one partition - consumer.incremental_unassign(&tpl).unwrap(); - assignments = consumer.assignment().unwrap(); - assert_eq!(assignments.count(), 1); - - // unassigning an non assigned partition should fail - let err = consumer.incremental_unassign(&tpl); - - assert_eq!( - err, - Err(KafkaError::Subscription("_INVALID_ARG".to_string())) - ) -} - -// All produced messages should be consumed. -#[tokio::test(flavor = "multi_thread")] -async fn test_produce_consume_with_timestamp() { - let _r = env_logger::try_init(); - - let topic_name = rand_test_topic("test_produce_consume_with_timestamp"); - let message_map = - populate_topic(&topic_name, 100, &value_fn, &key_fn, Some(0), Some(1111)).await; - let consumer = create_stream_consumer(&rand_test_group(), None); - consumer.subscribe(&[topic_name.as_str()]).unwrap(); - - let _consumer_future = consumer - .stream() - .take(100) - .for_each(|message| { - match message { - Ok(m) => { - let id = message_map[&(m.partition(), m.offset())]; - assert_eq!(m.timestamp(), Timestamp::CreateTime(1111)); - assert_eq!(m.payload_view::().unwrap().unwrap(), value_fn(id)); - assert_eq!(m.key_view::().unwrap().unwrap(), key_fn(id)); - } - Err(e) => panic!("Error receiving message: {:?}", e), - }; - future::ready(()) - }) - .await; - - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), Some(999_999)).await; - - // Lookup the offsets - let tpl = consumer - .offsets_for_timestamp(999_999, Duration::from_secs(10)) - .unwrap(); - let tp = tpl.find_partition(&topic_name, 0).unwrap(); - assert_eq!(tp.topic(), topic_name); - assert_eq!(tp.offset(), Offset::Offset(100)); - assert_eq!(tp.partition(), 0); - assert_eq!(tp.error(), Ok(())); -} - -// TODO: add check that commit cb gets called correctly -#[tokio::test(flavor = "multi_thread")] -async fn test_consumer_commit_message() { - let _r = env_logger::try_init(); - - let topic_name = rand_test_topic("test_consumer_commit_message"); - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; - populate_topic(&topic_name, 11, &value_fn, &key_fn, Some(1), None).await; - populate_topic(&topic_name, 12, &value_fn, &key_fn, Some(2), None).await; - let consumer = create_stream_consumer(&rand_test_group(), None); - consumer.subscribe(&[topic_name.as_str()]).unwrap(); - - let _consumer_future = consumer - .stream() - .take(33) - .for_each(|message| { - match message { - Ok(m) => { - if m.partition() == 1 { - consumer.commit_message(&m, CommitMode::Async).unwrap(); - } - } - Err(e) => panic!("error receiving message: {:?}", e), - }; - future::ready(()) - }) - .await; - - let timeout = Duration::from_secs(5); - assert_eq!( - consumer.fetch_watermarks(&topic_name, 0, timeout).unwrap(), - (0, 10) - ); - assert_eq!( - consumer.fetch_watermarks(&topic_name, 1, timeout).unwrap(), - (0, 11) - ); - assert_eq!( - consumer.fetch_watermarks(&topic_name, 2, timeout).unwrap(), - (0, 12) - ); - - let mut assignment = TopicPartitionList::new(); - assignment - .add_partition_offset(&topic_name, 0, Offset::Stored) - .unwrap(); - assignment - .add_partition_offset(&topic_name, 1, Offset::Stored) - .unwrap(); - assignment - .add_partition_offset(&topic_name, 2, Offset::Stored) - .unwrap(); - assert_eq!(assignment, consumer.assignment().unwrap()); - - let mut committed = TopicPartitionList::new(); - committed - .add_partition_offset(&topic_name, 0, Offset::Invalid) - .unwrap(); - committed - .add_partition_offset(&topic_name, 1, Offset::Offset(11)) - .unwrap(); - committed - .add_partition_offset(&topic_name, 2, Offset::Invalid) - .unwrap(); - assert_eq!(committed, consumer.committed(timeout).unwrap()); - - let mut position = TopicPartitionList::new(); - position - .add_partition_offset(&topic_name, 0, Offset::Offset(10)) - .unwrap(); - position - .add_partition_offset(&topic_name, 1, Offset::Offset(11)) - .unwrap(); - position - .add_partition_offset(&topic_name, 2, Offset::Offset(12)) - .unwrap(); - assert_eq!(position, consumer.position().unwrap()); -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_consumer_store_offset_commit() { - let _r = env_logger::try_init(); - - let topic_name = rand_test_topic("test_consumer_store_offset_commit"); - populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; - populate_topic(&topic_name, 11, &value_fn, &key_fn, Some(1), None).await; - populate_topic(&topic_name, 12, &value_fn, &key_fn, Some(2), None).await; - let mut config = HashMap::new(); - config.insert("enable.auto.offset.store", "false"); - config.insert("enable.partition.eof", "true"); - let consumer = create_stream_consumer(&rand_test_group(), Some(config)); - consumer.subscribe(&[topic_name.as_str()]).unwrap(); - - let _consumer_future = consumer - .stream() - .take(36) - .for_each(|message| { - match message { - Ok(m) => { - if m.partition() == 1 { - consumer.store_offset_from_message(&m).unwrap(); - } - } - Err(KafkaError::PartitionEOF(_)) => {} - Err(e) => panic!("Error receiving message: {:?}", e), - }; - future::ready(()) - }) - .await; - - // Commit the whole current state - consumer.commit_consumer_state(CommitMode::Sync).unwrap(); - - let timeout = Duration::from_secs(5); - assert_eq!( - consumer.fetch_watermarks(&topic_name, 0, timeout).unwrap(), - (0, 10) - ); - assert_eq!( - consumer.fetch_watermarks(&topic_name, 1, timeout).unwrap(), - (0, 11) - ); - assert_eq!( - consumer.fetch_watermarks(&topic_name, 2, timeout).unwrap(), - (0, 12) - ); - - let mut assignment = TopicPartitionList::new(); - assignment - .add_partition_offset(&topic_name, 0, Offset::Stored) - .unwrap(); - assignment - .add_partition_offset(&topic_name, 1, Offset::Stored) - .unwrap(); - assignment - .add_partition_offset(&topic_name, 2, Offset::Stored) - .unwrap(); - assert_eq!(assignment, consumer.assignment().unwrap()); - - let mut committed = TopicPartitionList::new(); - committed - .add_partition_offset(&topic_name, 0, Offset::Invalid) - .unwrap(); - committed - .add_partition_offset(&topic_name, 1, Offset::Offset(11)) - .unwrap(); - committed - .add_partition_offset(&topic_name, 2, Offset::Invalid) - .unwrap(); - assert_eq!(committed, consumer.committed(timeout).unwrap()); - - let mut position = TopicPartitionList::new(); - position - .add_partition_offset(&topic_name, 0, Offset::Offset(10)) - .unwrap(); - position - .add_partition_offset(&topic_name, 1, Offset::Offset(11)) - .unwrap(); - position - .add_partition_offset(&topic_name, 2, Offset::Offset(12)) - .unwrap(); - assert_eq!(position, consumer.position().unwrap()); -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_consumer_commit_metadata() -> Result<(), Box> { - let _ = env_logger::try_init(); - - let topic_name = rand_test_topic("test_consumer_commit_metadata"); - let group_name = rand_test_group(); - populate_topic(&topic_name, 10, &value_fn, &key_fn, None, None).await; - - let create_consumer = || async { - // Disable auto-commit so we can manually drive the commits. - let mut config = HashMap::new(); - config.insert("enable.auto.commit", "false"); - let consumer = create_stream_consumer(&group_name, Some(config)); - - // Subscribe to the topic and wait for at least one message, which - // ensures that the consumer group has been joined and such. - consumer.subscribe(&[topic_name.as_str()])?; - let _ = consumer.stream().next().await; - - Ok::<_, Box>(consumer) - }; - - // Create a topic partition list where each element has some associated - // metadata. - let tpl = { - let mut tpl = TopicPartitionList::new(); - let mut tpl1 = tpl.add_partition(&topic_name, 0); - tpl1.set_offset(Offset::Offset(1))?; - tpl1.set_metadata("one"); - let mut tpl2 = tpl.add_partition(&topic_name, 1); - tpl2.set_offset(Offset::Offset(1))?; - tpl2.set_metadata("two"); - let mut tpl3 = tpl.add_partition(&topic_name, 2); - tpl3.set_offset(Offset::Offset(1))?; - tpl3.set_metadata("three"); - tpl - }; - - // Ensure that the commit state immediately includes the metadata. - { - let consumer = create_consumer().await?; - consumer.commit(&tpl, CommitMode::Sync)?; - assert_eq!(consumer.committed(None)?, tpl); - } - - // Ensure that the commit state on a new consumer in the same group - // can see the same metadata. - { - let consumer = create_consumer().await?; - assert_eq!(consumer.committed(None)?, tpl); - } - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_consume_partition_order() { - let _r = env_logger::try_init(); - - let topic_name = rand_test_topic("test_consume_partition_order"); - populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(0), None).await; - populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(1), None).await; - populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(2), None).await; - - // Using partition queues should allow us to consume the partitions - // in a round-robin fashion. - { - let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); - let mut tpl = TopicPartitionList::new(); - tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) - .unwrap(); - tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) - .unwrap(); - tpl.add_partition_offset(&topic_name, 2, Offset::Beginning) - .unwrap(); - consumer.assign(&tpl).unwrap(); - - let mut partition_streams: Vec<_> = (0..3) - .map(|i| consumer.split_partition_queue(&topic_name, i).unwrap()) - .collect(); - - for _ in 0..4 { - let main_message = - time::timeout(Duration::from_millis(100), consumer.stream().next()).await; - assert!(main_message.is_err()); - - for (i, stream) in partition_streams.iter_mut().enumerate() { - let queue_message = stream.recv().await.unwrap(); - assert_eq!(queue_message.partition(), i as i32); - } - } - } - - // When not all partitions have been split into separate queues, the - // unsplit partitions should still be accessible via the main queue. - { - let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); - let mut tpl = TopicPartitionList::new(); - tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) - .unwrap(); - tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) - .unwrap(); - tpl.add_partition_offset(&topic_name, 2, Offset::Beginning) - .unwrap(); - consumer.assign(&tpl).unwrap(); - - let partition1 = consumer.split_partition_queue(&topic_name, 1).unwrap(); - - let mut i = 0; - while i < 5 { - if let Ok(m) = time::timeout(Duration::from_millis(1000), consumer.recv()).await { - // retry on transient errors until we get a message - let m = match m { - Err(KafkaError::MessageConsumption( - RDKafkaErrorCode::BrokerTransportFailure, - )) - | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::AllBrokersDown)) - | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::OperationTimedOut)) => { - continue; - } - Err(err) => { - panic!("Unexpected error receiving message: {:?}", err); - } - Ok(m) => m, - }; - let partition: i32 = m.partition(); - assert!(partition == 0 || partition == 2); - i += 1; - } else { - panic!("Timeout receiving message"); - } - - if let Ok(m) = time::timeout(Duration::from_millis(1000), partition1.recv()).await { - // retry on transient errors until we get a message - let m = match m { - Err(KafkaError::MessageConsumption( - RDKafkaErrorCode::BrokerTransportFailure, - )) - | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::AllBrokersDown)) - | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::OperationTimedOut)) => { - continue; - } - Err(err) => { - panic!("Unexpected error receiving message: {:?}", err); - } - Ok(m) => m, - }; - assert_eq!(m.partition(), 1); - i += 1; - } else { - panic!("Timeout receiving message"); - } - } - } - - // Sending the queue to another task that is likely to outlive the - // original thread should work. This is not idiomatic, as the consumer - // should be continuously polled to serve callbacks, but it should not panic - // or result in memory unsafety, etc. - { - let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); - let mut tpl = TopicPartitionList::new(); - tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) - .unwrap(); - consumer.assign(&tpl).unwrap(); - let stream = consumer.split_partition_queue(&topic_name, 0).unwrap(); - - let worker = tokio::spawn({ - async move { - for _ in 0..4 { - let stream_message = stream.recv().await.unwrap(); - assert_eq!(stream_message.partition(), 0); - } - } - }); - - let main_message = - time::timeout(Duration::from_millis(100), consumer.stream().next()).await; - assert!(main_message.is_err()); - - drop(consumer); - worker.await.unwrap(); - } -} +// #[tokio::test(flavor = "multi_thread")] +// async fn test_produce_consume_base_incremental_assign_and_unassign() { +// let _r = env_logger::try_init(); +// +// let topic_name = rand_test_topic("test_produce_consume_base_incremental_assign_and_unassign"); +// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; +// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(1), None).await; +// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(2), None).await; +// let consumer = create_stream_consumer(&rand_test_group(), None); +// +// // Adding a simple partition +// let mut tpl = TopicPartitionList::new(); +// tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) +// .unwrap(); +// consumer.incremental_assign(&tpl).unwrap(); +// let mut assignments = consumer.assignment().unwrap(); +// assert_eq!(assignments.count(), 1); +// +// // Adding another partition +// let mut tpl = TopicPartitionList::new(); +// tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) +// .unwrap(); +// consumer.incremental_assign(&tpl).unwrap(); +// assignments = consumer.assignment().unwrap(); +// assert_eq!(assignments.count(), 2); +// +// // Removing one partition +// consumer.incremental_unassign(&tpl).unwrap(); +// assignments = consumer.assignment().unwrap(); +// assert_eq!(assignments.count(), 1); +// +// // unassigning an non assigned partition should fail +// let err = consumer.incremental_unassign(&tpl); +// +// assert_eq!( +// err, +// Err(KafkaError::Subscription("_INVALID_ARG".to_string())) +// ) +// } +// +// // All produced messages should be consumed. +// #[tokio::test(flavor = "multi_thread")] +// async fn test_produce_consume_with_timestamp() { +// let _r = env_logger::try_init(); +// +// let topic_name = rand_test_topic("test_produce_consume_with_timestamp"); +// let message_map = +// populate_topic(&topic_name, 100, &value_fn, &key_fn, Some(0), Some(1111)).await; +// let consumer = create_stream_consumer(&rand_test_group(), None); +// consumer.subscribe(&[topic_name.as_str()]).unwrap(); +// +// let _consumer_future = consumer +// .stream() +// .take(100) +// .for_each(|message| { +// match message { +// Ok(m) => { +// let id = message_map[&(m.partition(), m.offset())]; +// assert_eq!(m.timestamp(), Timestamp::CreateTime(1111)); +// assert_eq!(m.payload_view::().unwrap().unwrap(), value_fn(id)); +// assert_eq!(m.key_view::().unwrap().unwrap(), key_fn(id)); +// } +// Err(e) => panic!("Error receiving message: {:?}", e), +// }; +// future::ready(()) +// }) +// .await; +// +// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), Some(999_999)).await; +// +// // Lookup the offsets +// let tpl = consumer +// .offsets_for_timestamp(999_999, Duration::from_secs(10)) +// .unwrap(); +// let tp = tpl.find_partition(&topic_name, 0).unwrap(); +// assert_eq!(tp.topic(), topic_name); +// assert_eq!(tp.offset(), Offset::Offset(100)); +// assert_eq!(tp.partition(), 0); +// assert_eq!(tp.error(), Ok(())); +// } +// +// // TODO: add check that commit cb gets called correctly +// #[tokio::test(flavor = "multi_thread")] +// async fn test_consumer_commit_message() { +// let _r = env_logger::try_init(); +// +// let topic_name = rand_test_topic("test_consumer_commit_message"); +// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; +// populate_topic(&topic_name, 11, &value_fn, &key_fn, Some(1), None).await; +// populate_topic(&topic_name, 12, &value_fn, &key_fn, Some(2), None).await; +// let consumer = create_stream_consumer(&rand_test_group(), None); +// consumer.subscribe(&[topic_name.as_str()]).unwrap(); +// +// let _consumer_future = consumer +// .stream() +// .take(33) +// .for_each(|message| { +// match message { +// Ok(m) => { +// if m.partition() == 1 { +// consumer.commit_message(&m, CommitMode::Async).unwrap(); +// } +// } +// Err(e) => panic!("error receiving message: {:?}", e), +// }; +// future::ready(()) +// }) +// .await; +// +// let timeout = Duration::from_secs(5); +// assert_eq!( +// consumer.fetch_watermarks(&topic_name, 0, timeout).unwrap(), +// (0, 10) +// ); +// assert_eq!( +// consumer.fetch_watermarks(&topic_name, 1, timeout).unwrap(), +// (0, 11) +// ); +// assert_eq!( +// consumer.fetch_watermarks(&topic_name, 2, timeout).unwrap(), +// (0, 12) +// ); +// +// let mut assignment = TopicPartitionList::new(); +// assignment +// .add_partition_offset(&topic_name, 0, Offset::Stored) +// .unwrap(); +// assignment +// .add_partition_offset(&topic_name, 1, Offset::Stored) +// .unwrap(); +// assignment +// .add_partition_offset(&topic_name, 2, Offset::Stored) +// .unwrap(); +// assert_eq!(assignment, consumer.assignment().unwrap()); +// +// let mut committed = TopicPartitionList::new(); +// committed +// .add_partition_offset(&topic_name, 0, Offset::Invalid) +// .unwrap(); +// committed +// .add_partition_offset(&topic_name, 1, Offset::Offset(11)) +// .unwrap(); +// committed +// .add_partition_offset(&topic_name, 2, Offset::Invalid) +// .unwrap(); +// assert_eq!(committed, consumer.committed(timeout).unwrap()); +// +// let mut position = TopicPartitionList::new(); +// position +// .add_partition_offset(&topic_name, 0, Offset::Offset(10)) +// .unwrap(); +// position +// .add_partition_offset(&topic_name, 1, Offset::Offset(11)) +// .unwrap(); +// position +// .add_partition_offset(&topic_name, 2, Offset::Offset(12)) +// .unwrap(); +// assert_eq!(position, consumer.position().unwrap()); +// } +// +// #[tokio::test(flavor = "multi_thread")] +// async fn test_consumer_store_offset_commit() { +// let _r = env_logger::try_init(); +// +// let topic_name = rand_test_topic("test_consumer_store_offset_commit"); +// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; +// populate_topic(&topic_name, 11, &value_fn, &key_fn, Some(1), None).await; +// populate_topic(&topic_name, 12, &value_fn, &key_fn, Some(2), None).await; +// let mut config = HashMap::new(); +// config.insert("enable.auto.offset.store", "false"); +// config.insert("enable.partition.eof", "true"); +// let consumer = create_stream_consumer(&rand_test_group(), Some(config)); +// consumer.subscribe(&[topic_name.as_str()]).unwrap(); +// +// let _consumer_future = consumer +// .stream() +// .take(36) +// .for_each(|message| { +// match message { +// Ok(m) => { +// if m.partition() == 1 { +// consumer.store_offset_from_message(&m).unwrap(); +// } +// } +// Err(KafkaError::PartitionEOF(_)) => {} +// Err(e) => panic!("Error receiving message: {:?}", e), +// }; +// future::ready(()) +// }) +// .await; +// +// // Commit the whole current state +// consumer.commit_consumer_state(CommitMode::Sync).unwrap(); +// +// let timeout = Duration::from_secs(5); +// assert_eq!( +// consumer.fetch_watermarks(&topic_name, 0, timeout).unwrap(), +// (0, 10) +// ); +// assert_eq!( +// consumer.fetch_watermarks(&topic_name, 1, timeout).unwrap(), +// (0, 11) +// ); +// assert_eq!( +// consumer.fetch_watermarks(&topic_name, 2, timeout).unwrap(), +// (0, 12) +// ); +// +// let mut assignment = TopicPartitionList::new(); +// assignment +// .add_partition_offset(&topic_name, 0, Offset::Stored) +// .unwrap(); +// assignment +// .add_partition_offset(&topic_name, 1, Offset::Stored) +// .unwrap(); +// assignment +// .add_partition_offset(&topic_name, 2, Offset::Stored) +// .unwrap(); +// assert_eq!(assignment, consumer.assignment().unwrap()); +// +// let mut committed = TopicPartitionList::new(); +// committed +// .add_partition_offset(&topic_name, 0, Offset::Invalid) +// .unwrap(); +// committed +// .add_partition_offset(&topic_name, 1, Offset::Offset(11)) +// .unwrap(); +// committed +// .add_partition_offset(&topic_name, 2, Offset::Invalid) +// .unwrap(); +// assert_eq!(committed, consumer.committed(timeout).unwrap()); +// +// let mut position = TopicPartitionList::new(); +// position +// .add_partition_offset(&topic_name, 0, Offset::Offset(10)) +// .unwrap(); +// position +// .add_partition_offset(&topic_name, 1, Offset::Offset(11)) +// .unwrap(); +// position +// .add_partition_offset(&topic_name, 2, Offset::Offset(12)) +// .unwrap(); +// assert_eq!(position, consumer.position().unwrap()); +// } +// +// #[tokio::test(flavor = "multi_thread")] +// async fn test_consumer_commit_metadata() -> Result<(), Box> { +// let _ = env_logger::try_init(); +// +// let topic_name = rand_test_topic("test_consumer_commit_metadata"); +// let group_name = rand_test_group(); +// populate_topic(&topic_name, 10, &value_fn, &key_fn, None, None).await; +// +// let create_consumer = || async { +// // Disable auto-commit so we can manually drive the commits. +// let mut config = HashMap::new(); +// config.insert("enable.auto.commit", "false"); +// let consumer = create_stream_consumer(&group_name, Some(config)); +// +// // Subscribe to the topic and wait for at least one message, which +// // ensures that the consumer group has been joined and such. +// consumer.subscribe(&[topic_name.as_str()])?; +// let _ = consumer.stream().next().await; +// +// Ok::<_, Box>(consumer) +// }; +// +// // Create a topic partition list where each element has some associated +// // metadata. +// let tpl = { +// let mut tpl = TopicPartitionList::new(); +// let mut tpl1 = tpl.add_partition(&topic_name, 0); +// tpl1.set_offset(Offset::Offset(1))?; +// tpl1.set_metadata("one"); +// let mut tpl2 = tpl.add_partition(&topic_name, 1); +// tpl2.set_offset(Offset::Offset(1))?; +// tpl2.set_metadata("two"); +// let mut tpl3 = tpl.add_partition(&topic_name, 2); +// tpl3.set_offset(Offset::Offset(1))?; +// tpl3.set_metadata("three"); +// tpl +// }; +// +// // Ensure that the commit state immediately includes the metadata. +// { +// let consumer = create_consumer().await?; +// consumer.commit(&tpl, CommitMode::Sync)?; +// assert_eq!(consumer.committed(None)?, tpl); +// } +// +// // Ensure that the commit state on a new consumer in the same group +// // can see the same metadata. +// { +// let consumer = create_consumer().await?; +// assert_eq!(consumer.committed(None)?, tpl); +// } +// +// Ok(()) +// } +// +// #[tokio::test(flavor = "multi_thread")] +// async fn test_consume_partition_order() { +// let _r = env_logger::try_init(); +// +// let topic_name = rand_test_topic("test_consume_partition_order"); +// populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(0), None).await; +// populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(1), None).await; +// populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(2), None).await; +// +// // Using partition queues should allow us to consume the partitions +// // in a round-robin fashion. +// { +// let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); +// let mut tpl = TopicPartitionList::new(); +// tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) +// .unwrap(); +// tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) +// .unwrap(); +// tpl.add_partition_offset(&topic_name, 2, Offset::Beginning) +// .unwrap(); +// consumer.assign(&tpl).unwrap(); +// +// let mut partition_streams: Vec<_> = (0..3) +// .map(|i| consumer.split_partition_queue(&topic_name, i).unwrap()) +// .collect(); +// +// for _ in 0..4 { +// let main_message = +// time::timeout(Duration::from_millis(100), consumer.stream().next()).await; +// assert!(main_message.is_err()); +// +// for (i, stream) in partition_streams.iter_mut().enumerate() { +// let queue_message = stream.recv().await.unwrap(); +// assert_eq!(queue_message.partition(), i as i32); +// } +// } +// } +// +// // When not all partitions have been split into separate queues, the +// // unsplit partitions should still be accessible via the main queue. +// { +// let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); +// let mut tpl = TopicPartitionList::new(); +// tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) +// .unwrap(); +// tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) +// .unwrap(); +// tpl.add_partition_offset(&topic_name, 2, Offset::Beginning) +// .unwrap(); +// consumer.assign(&tpl).unwrap(); +// +// let partition1 = consumer.split_partition_queue(&topic_name, 1).unwrap(); +// +// let mut i = 0; +// while i < 5 { +// if let Ok(m) = time::timeout(Duration::from_millis(1000), consumer.recv()).await { +// // retry on transient errors until we get a message +// let m = match m { +// Err(KafkaError::MessageConsumption( +// RDKafkaErrorCode::BrokerTransportFailure, +// )) +// | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::AllBrokersDown)) +// | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::OperationTimedOut)) => { +// continue; +// } +// Err(err) => { +// panic!("Unexpected error receiving message: {:?}", err); +// } +// Ok(m) => m, +// }; +// let partition: i32 = m.partition(); +// assert!(partition == 0 || partition == 2); +// i += 1; +// } else { +// panic!("Timeout receiving message"); +// } +// +// if let Ok(m) = time::timeout(Duration::from_millis(1000), partition1.recv()).await { +// // retry on transient errors until we get a message +// let m = match m { +// Err(KafkaError::MessageConsumption( +// RDKafkaErrorCode::BrokerTransportFailure, +// )) +// | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::AllBrokersDown)) +// | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::OperationTimedOut)) => { +// continue; +// } +// Err(err) => { +// panic!("Unexpected error receiving message: {:?}", err); +// } +// Ok(m) => m, +// }; +// assert_eq!(m.partition(), 1); +// i += 1; +// } else { +// panic!("Timeout receiving message"); +// } +// } +// } +// +// // Sending the queue to another task that is likely to outlive the +// // original thread should work. This is not idiomatic, as the consumer +// // should be continuously polled to serve callbacks, but it should not panic +// // or result in memory unsafety, etc. +// { +// let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); +// let mut tpl = TopicPartitionList::new(); +// tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) +// .unwrap(); +// consumer.assign(&tpl).unwrap(); +// let stream = consumer.split_partition_queue(&topic_name, 0).unwrap(); +// +// let worker = tokio::spawn({ +// async move { +// for _ in 0..4 { +// let stream_message = stream.recv().await.unwrap(); +// assert_eq!(stream_message.partition(), 0); +// } +// } +// }); +// +// let main_message = +// time::timeout(Duration::from_millis(100), consumer.stream().next()).await; +// assert!(main_message.is_err()); +// +// drop(consumer); +// worker.await.unwrap(); +// } +// } From e0104145ea65501455040ce0f14b719492c989c4 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 17 Oct 2025 18:13:54 -0400 Subject: [PATCH 23/56] test_produce_consume_base_incremental_assign_and_unassign --- tests/stream_consumers.rs | 121 ++++++++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 39 deletions(-) diff --git a/tests/stream_consumers.rs b/tests/stream_consumers.rs index b1839e17b..227962aea 100644 --- a/tests/stream_consumers.rs +++ b/tests/stream_consumers.rs @@ -335,45 +335,88 @@ async fn test_produce_consume_base_unassign() { assert_eq!(assignments.count(), 0); } -// #[tokio::test(flavor = "multi_thread")] -// async fn test_produce_consume_base_incremental_assign_and_unassign() { -// let _r = env_logger::try_init(); -// -// let topic_name = rand_test_topic("test_produce_consume_base_incremental_assign_and_unassign"); -// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; -// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(1), None).await; -// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(2), None).await; -// let consumer = create_stream_consumer(&rand_test_group(), None); -// -// // Adding a simple partition -// let mut tpl = TopicPartitionList::new(); -// tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) -// .unwrap(); -// consumer.incremental_assign(&tpl).unwrap(); -// let mut assignments = consumer.assignment().unwrap(); -// assert_eq!(assignments.count(), 1); -// -// // Adding another partition -// let mut tpl = TopicPartitionList::new(); -// tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) -// .unwrap(); -// consumer.incremental_assign(&tpl).unwrap(); -// assignments = consumer.assignment().unwrap(); -// assert_eq!(assignments.count(), 2); -// -// // Removing one partition -// consumer.incremental_unassign(&tpl).unwrap(); -// assignments = consumer.assignment().unwrap(); -// assert_eq!(assignments.count(), 1); -// -// // unassigning an non assigned partition should fail -// let err = consumer.incremental_unassign(&tpl); -// -// assert_eq!( -// err, -// Err(KafkaError::Subscription("_INVALID_ARG".to_string())) -// ) -// } +#[tokio::test(flavor = "multi_thread")] +async fn test_produce_consume_base_incremental_assign_and_unassign() { + init_test_logger(); + + // Get Kafka container context. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + let topic_name = rand_test_topic("test_produce_consume_base_incremental_assign_and_unassign"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client.create_topics(&new_topic_vec(&topic_name, Some(3)), &AdminOptions::default()).await.expect("could not create topics"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); + + let num_of_messages_to_send = 10usize; + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + Some(0), + ) + .await + .expect("Could not populate topic using Future producer"); + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + Some(1), + ) + .await + .expect("Could not populate topic using Future producer"); + topics::populate_topic_using_future_producer( + &producer, + &topic_name, + num_of_messages_to_send, + Some(2), + ) + .await + .expect("Could not populate topic using Future producer"); + + + // Adding a simple partition + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) + .unwrap(); + consumer.incremental_assign(&tpl).unwrap(); + let mut assignments = consumer.assignment().unwrap(); + assert_eq!(assignments.count(), 1); + + // Adding another partition + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) + .unwrap(); + consumer.incremental_assign(&tpl).unwrap(); + assignments = consumer.assignment().unwrap(); + assert_eq!(assignments.count(), 2); + + // Removing one partition + consumer.incremental_unassign(&tpl).unwrap(); + assignments = consumer.assignment().unwrap(); + assert_eq!(assignments.count(), 1); + + // unassigning an non assigned partition should fail + let err = consumer.incremental_unassign(&tpl); + + assert_eq!( + err, + Err(KafkaError::Subscription("_INVALID_ARG".to_string())) + ) +} // // // All produced messages should be consumed. // #[tokio::test(flavor = "multi_thread")] From 1ab92a22a3de2fe85ccdc934f17d8a223c80723c Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 17 Oct 2025 18:36:27 -0400 Subject: [PATCH 24/56] stream consumer tests. --- tests/stream_consumers.rs | 1008 +++++++++++++---------- tests/utils/admin.rs | 6 +- tests/utils/consumer/stream_consumer.rs | 25 +- tests/utils/mod.rs | 69 ++ 4 files changed, 662 insertions(+), 446 deletions(-) diff --git a/tests/stream_consumers.rs b/tests/stream_consumers.rs index 227962aea..c5030734e 100644 --- a/tests/stream_consumers.rs +++ b/tests/stream_consumers.rs @@ -1,53 +1,31 @@ //! Test data consumption using high level consumers. -use std::collections::HashMap; use std::error::Error; use std::sync::Arc; +use anyhow::Context; use futures::future; use futures::stream::StreamExt; use maplit::hashmap; use rdkafka_sys::RDKafkaErrorCode; use tokio::time::{self, Duration}; +use rdkafka::admin::AdminOptions; use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer}; use rdkafka::error::KafkaError; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::current_time_millis; use rdkafka::{Message, Timestamp}; -use rdkafka::admin::AdminOptions; use rdkafka_sys::types::RDKafkaConfRes; +use crate::utils::admin::new_topic_vec; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; use crate::utils::rand::*; use crate::utils::*; -use crate::utils::admin::new_topic_vec; mod utils; -// Create stream consumer for tests -fn create_stream_consumer( - group_id: &str, - config_overrides: Option>, -) -> StreamConsumer { - let cons_context = ConsumerTestContext { _n: 64 }; - create_stream_consumer_with_context(group_id, config_overrides, cons_context) -} - -fn create_stream_consumer_with_context( - group_id: &str, - config_overrides: Option>, - context: C, -) -> StreamConsumer -where - C: ConsumerContext + 'static, -{ - consumer_config(group_id, config_overrides) - .create_with_context(context) - .expect("Consumer creation failed") -} - #[tokio::test] async fn test_invalid_max_poll_interval() { let res: Result = consumer_config( @@ -155,8 +133,8 @@ async fn test_produce_consume_base_concurrent() { num_of_messages_to_send, None, ) - .await - .expect("Could not populate topic using Future producer"); + .await + .expect("Could not populate topic using Future producer"); // let message_map = populate_topic(&topic_name, 100, &value_fn, &key_fn, None, None).await; let consumer = Arc::new( consumer::stream_consumer::create_stream_consumer( @@ -203,7 +181,13 @@ async fn test_produce_consume_base_assign() { let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) .await .expect("Could not create admin client"); - admin_client.create_topics(&new_topic_vec(&topic_name, Some(3)), &AdminOptions::default()).await.expect("could not create topics"); + admin_client + .create_topics( + &new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topics"); let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) .await @@ -216,24 +200,24 @@ async fn test_produce_consume_base_assign() { num_of_messages_to_send, Some(0), ) - .await - .expect("Could not populate topic using Future producer"); + .await + .expect("Could not populate topic using Future producer"); topics::populate_topic_using_future_producer( &producer, &topic_name, num_of_messages_to_send, Some(1), ) - .await - .expect("Could not populate topic using Future producer"); + .await + .expect("Could not populate topic using Future producer"); topics::populate_topic_using_future_producer( &producer, &topic_name, num_of_messages_to_send, Some(2), ) - .await - .expect("Could not populate topic using Future producer"); + .await + .expect("Could not populate topic using Future producer"); let consumer = utils::consumer::stream_consumer::create_stream_consumer( &kafka_context.bootstrap_servers, @@ -280,7 +264,13 @@ async fn test_produce_consume_base_unassign() { let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) .await .expect("Could not create admin client"); - admin_client.create_topics(&new_topic_vec(&topic_name, Some(3)), &AdminOptions::default()).await.expect("could not create topics"); + admin_client + .create_topics( + &new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topics"); let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) .await @@ -290,8 +280,8 @@ async fn test_produce_consume_base_unassign() { &kafka_context.bootstrap_servers, Some(&rand_test_group()), ) - .await - .expect("could not create stream consumer"); + .await + .expect("could not create stream consumer"); let num_of_messages_to_send = 10usize; topics::populate_topic_using_future_producer( @@ -300,24 +290,24 @@ async fn test_produce_consume_base_unassign() { num_of_messages_to_send, Some(0), ) - .await - .expect("Could not populate topic using Future producer"); + .await + .expect("Could not populate topic using Future producer"); topics::populate_topic_using_future_producer( &producer, &topic_name, num_of_messages_to_send, Some(1), ) - .await - .expect("Could not populate topic using Future producer"); + .await + .expect("Could not populate topic using Future producer"); topics::populate_topic_using_future_producer( &producer, &topic_name, num_of_messages_to_send, Some(2), ) - .await - .expect("Could not populate topic using Future producer"); + .await + .expect("Could not populate topic using Future producer"); let mut tpl = TopicPartitionList::new(); tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) @@ -348,7 +338,13 @@ async fn test_produce_consume_base_incremental_assign_and_unassign() { let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) .await .expect("Could not create admin client"); - admin_client.create_topics(&new_topic_vec(&topic_name, Some(3)), &AdminOptions::default()).await.expect("could not create topics"); + admin_client + .create_topics( + &new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topics"); let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) .await @@ -358,8 +354,8 @@ async fn test_produce_consume_base_incremental_assign_and_unassign() { &kafka_context.bootstrap_servers, Some(&rand_test_group()), ) - .await - .expect("could not create stream consumer"); + .await + .expect("could not create stream consumer"); let num_of_messages_to_send = 10usize; topics::populate_topic_using_future_producer( @@ -368,25 +364,24 @@ async fn test_produce_consume_base_incremental_assign_and_unassign() { num_of_messages_to_send, Some(0), ) - .await - .expect("Could not populate topic using Future producer"); + .await + .expect("Could not populate topic using Future producer"); topics::populate_topic_using_future_producer( &producer, &topic_name, num_of_messages_to_send, Some(1), ) - .await - .expect("Could not populate topic using Future producer"); + .await + .expect("Could not populate topic using Future producer"); topics::populate_topic_using_future_producer( &producer, &topic_name, num_of_messages_to_send, Some(2), ) - .await - .expect("Could not populate topic using Future producer"); - + .await + .expect("Could not populate topic using Future producer"); // Adding a simple partition let mut tpl = TopicPartitionList::new(); @@ -417,393 +412,520 @@ async fn test_produce_consume_base_incremental_assign_and_unassign() { Err(KafkaError::Subscription("_INVALID_ARG".to_string())) ) } -// -// // All produced messages should be consumed. -// #[tokio::test(flavor = "multi_thread")] -// async fn test_produce_consume_with_timestamp() { -// let _r = env_logger::try_init(); -// -// let topic_name = rand_test_topic("test_produce_consume_with_timestamp"); -// let message_map = -// populate_topic(&topic_name, 100, &value_fn, &key_fn, Some(0), Some(1111)).await; -// let consumer = create_stream_consumer(&rand_test_group(), None); -// consumer.subscribe(&[topic_name.as_str()]).unwrap(); -// -// let _consumer_future = consumer -// .stream() -// .take(100) -// .for_each(|message| { -// match message { -// Ok(m) => { -// let id = message_map[&(m.partition(), m.offset())]; -// assert_eq!(m.timestamp(), Timestamp::CreateTime(1111)); -// assert_eq!(m.payload_view::().unwrap().unwrap(), value_fn(id)); -// assert_eq!(m.key_view::().unwrap().unwrap(), key_fn(id)); -// } -// Err(e) => panic!("Error receiving message: {:?}", e), -// }; -// future::ready(()) -// }) -// .await; -// -// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), Some(999_999)).await; -// -// // Lookup the offsets -// let tpl = consumer -// .offsets_for_timestamp(999_999, Duration::from_secs(10)) -// .unwrap(); -// let tp = tpl.find_partition(&topic_name, 0).unwrap(); -// assert_eq!(tp.topic(), topic_name); -// assert_eq!(tp.offset(), Offset::Offset(100)); -// assert_eq!(tp.partition(), 0); -// assert_eq!(tp.error(), Ok(())); -// } -// -// // TODO: add check that commit cb gets called correctly -// #[tokio::test(flavor = "multi_thread")] -// async fn test_consumer_commit_message() { -// let _r = env_logger::try_init(); -// -// let topic_name = rand_test_topic("test_consumer_commit_message"); -// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; -// populate_topic(&topic_name, 11, &value_fn, &key_fn, Some(1), None).await; -// populate_topic(&topic_name, 12, &value_fn, &key_fn, Some(2), None).await; -// let consumer = create_stream_consumer(&rand_test_group(), None); -// consumer.subscribe(&[topic_name.as_str()]).unwrap(); -// -// let _consumer_future = consumer -// .stream() -// .take(33) -// .for_each(|message| { -// match message { -// Ok(m) => { -// if m.partition() == 1 { -// consumer.commit_message(&m, CommitMode::Async).unwrap(); -// } -// } -// Err(e) => panic!("error receiving message: {:?}", e), -// }; -// future::ready(()) -// }) -// .await; -// -// let timeout = Duration::from_secs(5); -// assert_eq!( -// consumer.fetch_watermarks(&topic_name, 0, timeout).unwrap(), -// (0, 10) -// ); -// assert_eq!( -// consumer.fetch_watermarks(&topic_name, 1, timeout).unwrap(), -// (0, 11) -// ); -// assert_eq!( -// consumer.fetch_watermarks(&topic_name, 2, timeout).unwrap(), -// (0, 12) -// ); -// -// let mut assignment = TopicPartitionList::new(); -// assignment -// .add_partition_offset(&topic_name, 0, Offset::Stored) -// .unwrap(); -// assignment -// .add_partition_offset(&topic_name, 1, Offset::Stored) -// .unwrap(); -// assignment -// .add_partition_offset(&topic_name, 2, Offset::Stored) -// .unwrap(); -// assert_eq!(assignment, consumer.assignment().unwrap()); -// -// let mut committed = TopicPartitionList::new(); -// committed -// .add_partition_offset(&topic_name, 0, Offset::Invalid) -// .unwrap(); -// committed -// .add_partition_offset(&topic_name, 1, Offset::Offset(11)) -// .unwrap(); -// committed -// .add_partition_offset(&topic_name, 2, Offset::Invalid) -// .unwrap(); -// assert_eq!(committed, consumer.committed(timeout).unwrap()); -// -// let mut position = TopicPartitionList::new(); -// position -// .add_partition_offset(&topic_name, 0, Offset::Offset(10)) -// .unwrap(); -// position -// .add_partition_offset(&topic_name, 1, Offset::Offset(11)) -// .unwrap(); -// position -// .add_partition_offset(&topic_name, 2, Offset::Offset(12)) -// .unwrap(); -// assert_eq!(position, consumer.position().unwrap()); -// } -// -// #[tokio::test(flavor = "multi_thread")] -// async fn test_consumer_store_offset_commit() { -// let _r = env_logger::try_init(); -// -// let topic_name = rand_test_topic("test_consumer_store_offset_commit"); -// populate_topic(&topic_name, 10, &value_fn, &key_fn, Some(0), None).await; -// populate_topic(&topic_name, 11, &value_fn, &key_fn, Some(1), None).await; -// populate_topic(&topic_name, 12, &value_fn, &key_fn, Some(2), None).await; -// let mut config = HashMap::new(); -// config.insert("enable.auto.offset.store", "false"); -// config.insert("enable.partition.eof", "true"); -// let consumer = create_stream_consumer(&rand_test_group(), Some(config)); -// consumer.subscribe(&[topic_name.as_str()]).unwrap(); -// -// let _consumer_future = consumer -// .stream() -// .take(36) -// .for_each(|message| { -// match message { -// Ok(m) => { -// if m.partition() == 1 { -// consumer.store_offset_from_message(&m).unwrap(); -// } -// } -// Err(KafkaError::PartitionEOF(_)) => {} -// Err(e) => panic!("Error receiving message: {:?}", e), -// }; -// future::ready(()) -// }) -// .await; -// -// // Commit the whole current state -// consumer.commit_consumer_state(CommitMode::Sync).unwrap(); -// -// let timeout = Duration::from_secs(5); -// assert_eq!( -// consumer.fetch_watermarks(&topic_name, 0, timeout).unwrap(), -// (0, 10) -// ); -// assert_eq!( -// consumer.fetch_watermarks(&topic_name, 1, timeout).unwrap(), -// (0, 11) -// ); -// assert_eq!( -// consumer.fetch_watermarks(&topic_name, 2, timeout).unwrap(), -// (0, 12) -// ); -// -// let mut assignment = TopicPartitionList::new(); -// assignment -// .add_partition_offset(&topic_name, 0, Offset::Stored) -// .unwrap(); -// assignment -// .add_partition_offset(&topic_name, 1, Offset::Stored) -// .unwrap(); -// assignment -// .add_partition_offset(&topic_name, 2, Offset::Stored) -// .unwrap(); -// assert_eq!(assignment, consumer.assignment().unwrap()); -// -// let mut committed = TopicPartitionList::new(); -// committed -// .add_partition_offset(&topic_name, 0, Offset::Invalid) -// .unwrap(); -// committed -// .add_partition_offset(&topic_name, 1, Offset::Offset(11)) -// .unwrap(); -// committed -// .add_partition_offset(&topic_name, 2, Offset::Invalid) -// .unwrap(); -// assert_eq!(committed, consumer.committed(timeout).unwrap()); -// -// let mut position = TopicPartitionList::new(); -// position -// .add_partition_offset(&topic_name, 0, Offset::Offset(10)) -// .unwrap(); -// position -// .add_partition_offset(&topic_name, 1, Offset::Offset(11)) -// .unwrap(); -// position -// .add_partition_offset(&topic_name, 2, Offset::Offset(12)) -// .unwrap(); -// assert_eq!(position, consumer.position().unwrap()); -// } -// -// #[tokio::test(flavor = "multi_thread")] -// async fn test_consumer_commit_metadata() -> Result<(), Box> { -// let _ = env_logger::try_init(); -// -// let topic_name = rand_test_topic("test_consumer_commit_metadata"); -// let group_name = rand_test_group(); -// populate_topic(&topic_name, 10, &value_fn, &key_fn, None, None).await; -// -// let create_consumer = || async { -// // Disable auto-commit so we can manually drive the commits. -// let mut config = HashMap::new(); -// config.insert("enable.auto.commit", "false"); -// let consumer = create_stream_consumer(&group_name, Some(config)); -// -// // Subscribe to the topic and wait for at least one message, which -// // ensures that the consumer group has been joined and such. -// consumer.subscribe(&[topic_name.as_str()])?; -// let _ = consumer.stream().next().await; -// -// Ok::<_, Box>(consumer) -// }; -// -// // Create a topic partition list where each element has some associated -// // metadata. -// let tpl = { -// let mut tpl = TopicPartitionList::new(); -// let mut tpl1 = tpl.add_partition(&topic_name, 0); -// tpl1.set_offset(Offset::Offset(1))?; -// tpl1.set_metadata("one"); -// let mut tpl2 = tpl.add_partition(&topic_name, 1); -// tpl2.set_offset(Offset::Offset(1))?; -// tpl2.set_metadata("two"); -// let mut tpl3 = tpl.add_partition(&topic_name, 2); -// tpl3.set_offset(Offset::Offset(1))?; -// tpl3.set_metadata("three"); -// tpl -// }; -// -// // Ensure that the commit state immediately includes the metadata. -// { -// let consumer = create_consumer().await?; -// consumer.commit(&tpl, CommitMode::Sync)?; -// assert_eq!(consumer.committed(None)?, tpl); -// } -// -// // Ensure that the commit state on a new consumer in the same group -// // can see the same metadata. -// { -// let consumer = create_consumer().await?; -// assert_eq!(consumer.committed(None)?, tpl); -// } -// -// Ok(()) -// } -// -// #[tokio::test(flavor = "multi_thread")] -// async fn test_consume_partition_order() { -// let _r = env_logger::try_init(); -// -// let topic_name = rand_test_topic("test_consume_partition_order"); -// populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(0), None).await; -// populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(1), None).await; -// populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(2), None).await; -// -// // Using partition queues should allow us to consume the partitions -// // in a round-robin fashion. -// { -// let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); -// let mut tpl = TopicPartitionList::new(); -// tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) -// .unwrap(); -// tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) -// .unwrap(); -// tpl.add_partition_offset(&topic_name, 2, Offset::Beginning) -// .unwrap(); -// consumer.assign(&tpl).unwrap(); -// -// let mut partition_streams: Vec<_> = (0..3) -// .map(|i| consumer.split_partition_queue(&topic_name, i).unwrap()) -// .collect(); -// -// for _ in 0..4 { -// let main_message = -// time::timeout(Duration::from_millis(100), consumer.stream().next()).await; -// assert!(main_message.is_err()); -// -// for (i, stream) in partition_streams.iter_mut().enumerate() { -// let queue_message = stream.recv().await.unwrap(); -// assert_eq!(queue_message.partition(), i as i32); -// } -// } -// } -// -// // When not all partitions have been split into separate queues, the -// // unsplit partitions should still be accessible via the main queue. -// { -// let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); -// let mut tpl = TopicPartitionList::new(); -// tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) -// .unwrap(); -// tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) -// .unwrap(); -// tpl.add_partition_offset(&topic_name, 2, Offset::Beginning) -// .unwrap(); -// consumer.assign(&tpl).unwrap(); -// -// let partition1 = consumer.split_partition_queue(&topic_name, 1).unwrap(); -// -// let mut i = 0; -// while i < 5 { -// if let Ok(m) = time::timeout(Duration::from_millis(1000), consumer.recv()).await { -// // retry on transient errors until we get a message -// let m = match m { -// Err(KafkaError::MessageConsumption( -// RDKafkaErrorCode::BrokerTransportFailure, -// )) -// | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::AllBrokersDown)) -// | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::OperationTimedOut)) => { -// continue; -// } -// Err(err) => { -// panic!("Unexpected error receiving message: {:?}", err); -// } -// Ok(m) => m, -// }; -// let partition: i32 = m.partition(); -// assert!(partition == 0 || partition == 2); -// i += 1; -// } else { -// panic!("Timeout receiving message"); -// } -// -// if let Ok(m) = time::timeout(Duration::from_millis(1000), partition1.recv()).await { -// // retry on transient errors until we get a message -// let m = match m { -// Err(KafkaError::MessageConsumption( -// RDKafkaErrorCode::BrokerTransportFailure, -// )) -// | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::AllBrokersDown)) -// | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::OperationTimedOut)) => { -// continue; -// } -// Err(err) => { -// panic!("Unexpected error receiving message: {:?}", err); -// } -// Ok(m) => m, -// }; -// assert_eq!(m.partition(), 1); -// i += 1; -// } else { -// panic!("Timeout receiving message"); -// } -// } -// } -// -// // Sending the queue to another task that is likely to outlive the -// // original thread should work. This is not idiomatic, as the consumer -// // should be continuously polled to serve callbacks, but it should not panic -// // or result in memory unsafety, etc. -// { -// let consumer = Arc::new(create_stream_consumer(&rand_test_group(), None)); -// let mut tpl = TopicPartitionList::new(); -// tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) -// .unwrap(); -// consumer.assign(&tpl).unwrap(); -// let stream = consumer.split_partition_queue(&topic_name, 0).unwrap(); -// -// let worker = tokio::spawn({ -// async move { -// for _ in 0..4 { -// let stream_message = stream.recv().await.unwrap(); -// assert_eq!(stream_message.partition(), 0); -// } -// } -// }); -// -// let main_message = -// time::timeout(Duration::from_millis(100), consumer.stream().next()).await; -// assert!(main_message.is_err()); -// -// drop(consumer); -// worker.await.unwrap(); -// } -// } + +// All produced messages should be consumed. +#[tokio::test(flavor = "multi_thread")] +async fn test_produce_consume_with_timestamp() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_produce_consume_with_timestamp"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + + let message_map = produce_messages_with_timestamp(&producer, &topic_name, 100, 0, 1111).await; + + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); + consumer.subscribe(&[topic_name.as_str()]).unwrap(); + + let _consumer_future = consumer + .stream() + .take(100) + .for_each(|message| { + match message { + Ok(m) => { + let id = message_map[&(m.partition(), m.offset())]; + assert_eq!(m.timestamp(), Timestamp::CreateTime(1111)); + assert_eq!(m.payload_view::().unwrap().unwrap(), value_fn(id)); + assert_eq!(m.key_view::().unwrap().unwrap(), key_fn(id)); + } + Err(e) => panic!("Error receiving message: {:?}", e), + }; + future::ready(()) + }) + .await; + + let _ = produce_messages_with_timestamp(&producer, &topic_name, 10, 0, 999_999).await; + + // Lookup the offsets + let tpl = consumer + .offsets_for_timestamp(999_999, Duration::from_secs(10)) + .unwrap(); + let tp = tpl.find_partition(&topic_name, 0).unwrap(); + assert_eq!(tp.topic(), topic_name); + assert_eq!(tp.offset(), Offset::Offset(100)); + assert_eq!(tp.partition(), 0); + assert_eq!(tp.error(), Ok(())); +} + +// TODO: add check that commit cb gets called correctly +#[tokio::test(flavor = "multi_thread")] +async fn test_consumer_commit_message() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_consumer_commit_message"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topics"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + + let _ = produce_messages_to_partition(&producer, &topic_name, 10, 0).await; + let _ = produce_messages_to_partition(&producer, &topic_name, 11, 1).await; + let _ = produce_messages_to_partition(&producer, &topic_name, 12, 2).await; + + let group_name = rand_test_group(); + let consumer = utils::consumer::stream_consumer::create_stream_consumer_with_options( + &kafka_context.bootstrap_servers, + &group_name, + &[], + ) + .await + .expect("could not create stream consumer"); + consumer.subscribe(&[topic_name.as_str()]).unwrap(); + + let _consumer_future = consumer + .stream() + .take(33) + .for_each(|message| { + match message { + Ok(m) => { + if m.partition() == 1 { + consumer.commit_message(&m, CommitMode::Async).unwrap(); + } + } + Err(e) => panic!("error receiving message: {:?}", e), + }; + future::ready(()) + }) + .await; + + let timeout = Duration::from_secs(5); + assert_eq!( + consumer.fetch_watermarks(&topic_name, 0, timeout).unwrap(), + (0, 10) + ); + assert_eq!( + consumer.fetch_watermarks(&topic_name, 1, timeout).unwrap(), + (0, 11) + ); + assert_eq!( + consumer.fetch_watermarks(&topic_name, 2, timeout).unwrap(), + (0, 12) + ); + + let mut assignment = TopicPartitionList::new(); + assignment + .add_partition_offset(&topic_name, 0, Offset::Stored) + .unwrap(); + assignment + .add_partition_offset(&topic_name, 1, Offset::Stored) + .unwrap(); + assignment + .add_partition_offset(&topic_name, 2, Offset::Stored) + .unwrap(); + assert_eq!(assignment, consumer.assignment().unwrap()); + + let mut committed = TopicPartitionList::new(); + committed + .add_partition_offset(&topic_name, 0, Offset::Invalid) + .unwrap(); + committed + .add_partition_offset(&topic_name, 1, Offset::Offset(11)) + .unwrap(); + committed + .add_partition_offset(&topic_name, 2, Offset::Invalid) + .unwrap(); + assert_eq!(committed, consumer.committed(timeout).unwrap()); + + let mut position = TopicPartitionList::new(); + position + .add_partition_offset(&topic_name, 0, Offset::Offset(10)) + .unwrap(); + position + .add_partition_offset(&topic_name, 1, Offset::Offset(11)) + .unwrap(); + position + .add_partition_offset(&topic_name, 2, Offset::Offset(12)) + .unwrap(); + assert_eq!(position, consumer.position().unwrap()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_consumer_store_offset_commit() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_consumer_store_offset_commit"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topics"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + + let _ = produce_messages_to_partition(&producer, &topic_name, 10, 0).await; + let _ = produce_messages_to_partition(&producer, &topic_name, 11, 1).await; + let _ = produce_messages_to_partition(&producer, &topic_name, 12, 2).await; + + let group_name = rand_test_group(); + let consumer = utils::consumer::stream_consumer::create_stream_consumer_with_options( + &kafka_context.bootstrap_servers, + &group_name, + &[ + ("enable.auto.offset.store", "false"), + ("enable.partition.eof", "true"), + ], + ) + .await + .expect("could not create stream consumer"); + consumer.subscribe(&[topic_name.as_str()]).unwrap(); + + let _consumer_future = consumer + .stream() + .take(36) + .for_each(|message| { + match message { + Ok(m) => { + if m.partition() == 1 { + consumer.store_offset_from_message(&m).unwrap(); + } + } + Err(KafkaError::PartitionEOF(_)) => {} + Err(e) => panic!("Error receiving message: {:?}", e), + }; + future::ready(()) + }) + .await; + + // Commit the whole current state + consumer.commit_consumer_state(CommitMode::Sync).unwrap(); + + let timeout = Duration::from_secs(5); + assert_eq!( + consumer.fetch_watermarks(&topic_name, 0, timeout).unwrap(), + (0, 10) + ); + assert_eq!( + consumer.fetch_watermarks(&topic_name, 1, timeout).unwrap(), + (0, 11) + ); + assert_eq!( + consumer.fetch_watermarks(&topic_name, 2, timeout).unwrap(), + (0, 12) + ); + + let mut assignment = TopicPartitionList::new(); + assignment + .add_partition_offset(&topic_name, 0, Offset::Stored) + .unwrap(); + assignment + .add_partition_offset(&topic_name, 1, Offset::Stored) + .unwrap(); + assignment + .add_partition_offset(&topic_name, 2, Offset::Stored) + .unwrap(); + assert_eq!(assignment, consumer.assignment().unwrap()); + + let mut committed = TopicPartitionList::new(); + committed + .add_partition_offset(&topic_name, 0, Offset::Invalid) + .unwrap(); + committed + .add_partition_offset(&topic_name, 1, Offset::Offset(11)) + .unwrap(); + committed + .add_partition_offset(&topic_name, 2, Offset::Invalid) + .unwrap(); + assert_eq!(committed, consumer.committed(timeout).unwrap()); + + let mut position = TopicPartitionList::new(); + position + .add_partition_offset(&topic_name, 0, Offset::Offset(10)) + .unwrap(); + position + .add_partition_offset(&topic_name, 1, Offset::Offset(11)) + .unwrap(); + position + .add_partition_offset(&topic_name, 2, Offset::Offset(12)) + .unwrap(); + assert_eq!(position, consumer.position().unwrap()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_consumer_commit_metadata() -> Result<(), Box> { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_consumer_commit_metadata"); + let group_name = rand_test_group(); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topics"); + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + let _ = produce_messages_to_partition(&producer, &topic_name, 4, 0).await; + let _ = produce_messages_to_partition(&producer, &topic_name, 4, 1).await; + let _ = produce_messages_to_partition(&producer, &topic_name, 4, 2).await; + + let create_consumer = || async { + let consumer = utils::consumer::stream_consumer::create_stream_consumer_with_options( + &kafka_context.bootstrap_servers, + &group_name, + &[], + ) + .await + .context("failed to create stream consumer")?; + + consumer + .subscribe(&[topic_name.as_str()]) + .context("failed to subscribe to topic")?; + let _ = consumer.stream().next().await; + + Ok::<_, Box>(consumer) + }; + + // Create a topic partition list where each element has some associated + // metadata. + let tpl = { + let mut tpl = TopicPartitionList::new(); + let mut tpl1 = tpl.add_partition(&topic_name, 0); + tpl1.set_offset(Offset::Offset(1))?; + tpl1.set_metadata("one"); + let mut tpl2 = tpl.add_partition(&topic_name, 1); + tpl2.set_offset(Offset::Offset(1))?; + tpl2.set_metadata("two"); + let mut tpl3 = tpl.add_partition(&topic_name, 2); + tpl3.set_offset(Offset::Offset(1))?; + tpl3.set_metadata("three"); + tpl + }; + + // Ensure that the commit state immediately includes the metadata. + { + let consumer = create_consumer().await?; + consumer.commit(&tpl, CommitMode::Sync)?; + assert_eq!(consumer.committed(None)?, tpl); + } + + // Ensure that the commit state on a new consumer in the same group + // can see the same metadata. + { + let consumer = create_consumer().await?; + assert_eq!(consumer.committed(None)?, tpl); + } + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_consume_partition_order() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_consume_partition_order"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topics"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + let _ = produce_messages_to_partition(&producer, &topic_name, 4, 0).await; + let _ = produce_messages_to_partition(&producer, &topic_name, 4, 1).await; + let _ = produce_messages_to_partition(&producer, &topic_name, 4, 2).await; + + // Using partition queues should allow us to consume the partitions + // in a round-robin fashion. + { + let consumer = Arc::new( + utils::consumer::stream_consumer::create_stream_consumer_with_options( + &kafka_context.bootstrap_servers, + &rand_test_group(), + &[], + ) + .await + .expect("could not create stream consumer"), + ); + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) + .unwrap(); + tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) + .unwrap(); + tpl.add_partition_offset(&topic_name, 2, Offset::Beginning) + .unwrap(); + consumer.assign(&tpl).unwrap(); + + let mut partition_streams: Vec<_> = (0..3) + .map(|i| consumer.split_partition_queue(&topic_name, i).unwrap()) + .collect(); + + for _ in 0..4 { + let main_message = + time::timeout(Duration::from_millis(100), consumer.stream().next()).await; + assert!(main_message.is_err()); + + for (i, stream) in partition_streams.iter_mut().enumerate() { + let queue_message = stream.recv().await.unwrap(); + assert_eq!(queue_message.partition(), i as i32); + } + } + } + + // When not all partitions have been split into separate queues, the + // unsplit partitions should still be accessible via the main queue. + { + let consumer = Arc::new( + utils::consumer::stream_consumer::create_stream_consumer_with_options( + &kafka_context.bootstrap_servers, + &rand_test_group(), + &[], + ) + .await + .expect("could not create stream consumer"), + ); + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) + .unwrap(); + tpl.add_partition_offset(&topic_name, 1, Offset::Beginning) + .unwrap(); + tpl.add_partition_offset(&topic_name, 2, Offset::Beginning) + .unwrap(); + consumer.assign(&tpl).unwrap(); + + let partition1 = consumer.split_partition_queue(&topic_name, 1).unwrap(); + + let mut i = 0; + while i < 5 { + if let Ok(m) = time::timeout(Duration::from_millis(1000), consumer.recv()).await { + // retry on transient errors until we get a message + let m = match m { + Err(KafkaError::MessageConsumption( + RDKafkaErrorCode::BrokerTransportFailure, + )) + | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::AllBrokersDown)) + | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::OperationTimedOut)) => { + continue; + } + Err(err) => { + panic!("Unexpected error receiving message: {:?}", err); + } + Ok(m) => m, + }; + let partition: i32 = m.partition(); + assert!(partition == 0 || partition == 2); + i += 1; + } else { + panic!("Timeout receiving message"); + } + + if let Ok(m) = time::timeout(Duration::from_millis(1000), partition1.recv()).await { + // retry on transient errors until we get a message + let m = match m { + Err(KafkaError::MessageConsumption( + RDKafkaErrorCode::BrokerTransportFailure, + )) + | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::AllBrokersDown)) + | Err(KafkaError::MessageConsumption(RDKafkaErrorCode::OperationTimedOut)) => { + continue; + } + Err(err) => { + panic!("Unexpected error receiving message: {:?}", err); + } + Ok(m) => m, + }; + assert_eq!(m.partition(), 1); + i += 1; + } else { + panic!("Timeout receiving message"); + } + } + } + + // Sending the queue to another task that is likely to outlive the + // original thread should work. This is not idiomatic, as the consumer + // should be continuously polled to serve callbacks, but it should not panic + // or result in memory unsafety, etc. + { + let consumer = Arc::new( + utils::consumer::stream_consumer::create_stream_consumer_with_options( + &kafka_context.bootstrap_servers, + &rand_test_group(), + &[], + ) + .await + .expect("could not create stream consumer"), + ); + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) + .unwrap(); + consumer.assign(&tpl).unwrap(); + let stream = consumer.split_partition_queue(&topic_name, 0).unwrap(); + + let worker = tokio::spawn({ + async move { + for _ in 0..4 { + let stream_message = stream.recv().await.unwrap(); + assert_eq!(stream_message.partition(), 0); + } + } + }); + + let main_message = + time::timeout(Duration::from_millis(100), consumer.stream().next()).await; + assert!(main_message.is_err()); + + drop(consumer); + worker.await.unwrap(); + } +} diff --git a/tests/utils/admin.rs b/tests/utils/admin.rs index 9de144d72..2d9e0bb0e 100644 --- a/tests/utils/admin.rs +++ b/tests/utils/admin.rs @@ -29,6 +29,10 @@ pub async fn create_topic( } pub fn new_topic_vec(topic_name: &'_ str, num_partitions: Option) -> Vec> { - let new_topic = NewTopic::new(&topic_name, num_partitions.unwrap_or(1), TopicReplication::Fixed(1)); + let new_topic = NewTopic::new( + &topic_name, + num_partitions.unwrap_or(1), + TopicReplication::Fixed(1), + ); vec![new_topic] } diff --git a/tests/utils/consumer/stream_consumer.rs b/tests/utils/consumer/stream_consumer.rs index 51201873f..9e94d3a96 100644 --- a/tests/utils/consumer/stream_consumer.rs +++ b/tests/utils/consumer/stream_consumer.rs @@ -1,7 +1,7 @@ use anyhow::Context; -use rdkafka::ClientConfig; use rdkafka::config::FromClientConfig; use rdkafka::consumer::StreamConsumer; +use rdkafka::ClientConfig; pub async fn create_stream_consumer( bootstrap_server: &str, @@ -14,6 +14,27 @@ pub async fn create_stream_consumer( client_config.set("group.id", group); } - let stream_consumer = StreamConsumer::from_config(&client_config).context("failed to create stream consumer")?; + let stream_consumer = + StreamConsumer::from_config(&client_config).context("failed to create stream consumer")?; + Ok(stream_consumer) +} + +pub async fn create_stream_consumer_with_options( + bootstrap_server: &str, + consumer_group: &str, + options: &[(&str, &str)], +) -> anyhow::Result { + let mut client_config = ClientConfig::default(); + client_config.set("bootstrap.servers", bootstrap_server); + client_config.set("group.id", consumer_group); + client_config.set("auto.offset.reset", "earliest"); + client_config.set("enable.auto.commit", "false"); + + for (key, value) in options { + client_config.set(*key, *value); + } + + let stream_consumer = + StreamConsumer::from_config(&client_config).context("failed to create stream consumer")?; Ok(stream_consumer) } diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index ca9f496f2..e68caa5cf 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -129,6 +129,75 @@ where message_map } +pub async fn produce_messages_with_timestamp( + producer: &FutureProducer, + topic_name: &str, + count: usize, + partition: i32, + timestamp: i64, +) -> HashMap<(i32, i64), i32> { + produce_messages( + producer, + topic_name, + count, + Some(partition), + Some(timestamp), + ) + .await +} + +pub async fn produce_messages_to_partition( + producer: &FutureProducer, + topic_name: &str, + count: usize, + partition: i32, +) -> HashMap<(i32, i64), i32> { + produce_messages(producer, topic_name, count, Some(partition), None).await +} + +pub async fn produce_messages( + producer: &FutureProducer, + topic_name: &str, + count: usize, + partition: Option, + timestamp: Option, +) -> HashMap<(i32, i64), i32> { + let mut inflight = Vec::with_capacity(count); + + for idx in 0..count { + let id = idx as i32; + let payload = value_fn(id); + let key = key_fn(id); + let mut record = FutureRecord::to(topic_name).payload(&payload).key(&key); + if let Some(partition) = partition { + record = record.partition(partition); + } + if let Some(timestamp) = timestamp { + record = record.timestamp(timestamp); + } + let delivery_future = producer + .send_result(record) + .expect("failed to enqueue message"); + inflight.push((id, payload, key, delivery_future)); + } + + let mut message_map = HashMap::new(); + + for (id, _payload, _key, delivery_future) in inflight { + match delivery_future + .await + .expect("producer unexpectedly dropped") + { + Ok(delivery) => { + message_map.insert((delivery.partition, delivery.offset), id); + } + Err((error, _message)) => panic!("Delivery failed: {}", error), + }; + } + + message_map +} + pub fn value_fn(id: i32) -> String { format!("Message {}", id) } From 7f501ee2cf8bd092ae6acc8a46c4a989b045e4bf Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 17 Oct 2025 18:50:41 -0400 Subject: [PATCH 25/56] future_producer tests --- ...t_high_producers.rs => future_producer.rs} | 75 +++++++++++++------ 1 file changed, 54 insertions(+), 21 deletions(-) rename tests/{test_high_producers.rs => future_producer.rs} (74%) diff --git a/tests/test_high_producers.rs b/tests/future_producer.rs similarity index 74% rename from tests/test_high_producers.rs rename to tests/future_producer.rs index d12c7d1ee..3a0699839 100644 --- a/tests/test_high_producers.rs +++ b/tests/future_producer.rs @@ -1,10 +1,10 @@ //! Test data production using high level producers. -use std::collections::HashMap; use std::time::{Duration, Instant}; use futures::stream::{FuturesUnordered, StreamExt}; +use rdkafka::admin::AdminOptions; use rdkafka::client::DefaultClientContext; use rdkafka::config::ClientConfig; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; @@ -13,25 +13,36 @@ use rdkafka::producer::{FutureProducer, FutureRecord, Producer}; use rdkafka::util::Timeout; use rdkafka::Timestamp; +use crate::utils::admin; +use crate::utils::containers::KafkaContext; +use crate::utils::logging::init_test_logger; +use crate::utils::producer; use crate::utils::rand::*; mod utils; -fn future_producer(config_overrides: HashMap<&str, &str>) -> FutureProducer { - let mut config = ClientConfig::new(); - config - .set("bootstrap.servers", "localhost") - .set("message.timeout.ms", "5000"); - for (key, value) in config_overrides { - config.set(key, value); - } - config.create().expect("Failed to create producer") -} - #[tokio::test] async fn test_future_producer_send() { - let producer = future_producer(HashMap::new()); + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_future_producer_send"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); let results: FuturesUnordered<_> = (0..10) .map(|_| { @@ -57,11 +68,14 @@ async fn test_future_producer_send_full() { // Connect to a nonexistent Kafka broker with a long message timeout and a // tiny producer queue, so we can fill up the queue for a while by sending a // single message. - let mut config = HashMap::new(); - config.insert("bootstrap.servers", ""); - config.insert("message.timeout.ms", "5000"); - config.insert("queue.buffering.max.messages", "1"); - let producer = &future_producer(config); + let mut config = ClientConfig::new(); + config + .set("bootstrap.servers", "") + .set("message.timeout.ms", "5000") + .set("queue.buffering.max.messages", "1"); + let producer: FutureProducer = + config.create().expect("Failed to create producer"); + let producer = &producer; let topic_name = &rand_test_topic("test_future_producer_send_full"); // Fill up the queue. @@ -97,10 +111,29 @@ async fn test_future_producer_send_full() { #[tokio::test] async fn test_future_producer_send_fail() { - let producer = future_producer(HashMap::new()); + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_future_producer_send_fail"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); let future = producer.send( - FutureRecord::to("topic") + FutureRecord::to(&topic_name) .payload("payload") .key("key") .partition(100) // Fail @@ -128,7 +161,7 @@ async fn test_future_producer_send_fail() { kafka_error.to_string(), "Message production error: UnknownPartition (Local: Unknown partition)" ); - assert_eq!(owned_message.topic(), "topic"); + assert_eq!(owned_message.topic(), topic_name.as_str()); let headers = owned_message.headers().unwrap(); assert_eq!(headers.count(), 3); assert_eq!( From 3ee35514cd326404bfefb3fd8f48e2b81ebc3f63 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 17 Oct 2025 19:38:16 -0400 Subject: [PATCH 26/56] More test updates. --- ...test_low_consumers.rs => base_consumer.rs} | 215 +++++++++--- ...test_low_producers.rs => base_producer.rs} | 307 ++++++++++++------ tests/{test_metadata.rs => metadata.rs} | 107 ++++-- ...ition_list.rs => topic_partition_lists.rs} | 2 +- .../{test_transactions.rs => transactions.rs} | 134 ++++++-- tests/utils/consumer/mod.rs | 29 ++ tests/utils/producer/base_producer.rs | 48 ++- 7 files changed, 648 insertions(+), 194 deletions(-) rename tests/{test_low_consumers.rs => base_consumer.rs} (73%) rename tests/{test_low_producers.rs => base_producer.rs} (66%) rename tests/{test_metadata.rs => metadata.rs} (59%) rename tests/{test_topic_partition_list.rs => topic_partition_lists.rs} (92%) rename tests/{test_transactions.rs => transactions.rs} (55%) diff --git a/tests/test_low_consumers.rs b/tests/base_consumer.rs similarity index 73% rename from tests/test_low_consumers.rs rename to tests/base_consumer.rs index 307e2d824..3b0e4f61c 100644 --- a/tests/test_low_consumers.rs +++ b/tests/base_consumer.rs @@ -7,39 +7,56 @@ use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; +use rdkafka::admin::AdminOptions; use rdkafka::consumer::{BaseConsumer, Consumer, ConsumerContext}; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::{current_time_millis, Timeout}; use rdkafka::{ClientConfig, Message, Timestamp}; +use crate::utils::admin; +use crate::utils::containers::KafkaContext; +use crate::utils::logging::init_test_logger; +use crate::utils::producer; use crate::utils::rand::*; use crate::utils::*; mod utils; -fn create_base_consumer( - group_id: &str, - config_overrides: Option>, -) -> BaseConsumer { - consumer_config(group_id, config_overrides) - .create_with_context(ConsumerTestContext { _n: 64 }) - .expect("Consumer creation failed") -} - // Seeking should allow replaying messages and skipping messages. #[tokio::test] async fn test_produce_consume_seek() { - let _r = env_logger::try_init(); + init_test_logger(); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_produce_consume_seek"); - populate_topic(&topic_name, 5, &value_fn, &key_fn, Some(0), None).await; - let consumer = create_base_consumer(&rand_test_group(), None); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + produce_messages_to_partition(&producer, &topic_name, 5, 0).await; + + let group_id = rand_test_group(); + let consumer = + utils::consumer::create_base_consumer(&kafka_context.bootstrap_servers, &group_id, None) + .expect("could not create base consumer"); consumer.subscribe(&[topic_name.as_str()]).unwrap(); for (i, message) in consumer.iter().take(3).enumerate() { match message { - Ok(message) => assert_eq!(dbg!(message.offset()), i as i64), + Ok(message) => assert_eq!(message.offset(), i as i64), Err(e) => panic!("Error receiving message: {:?}", e), } } @@ -95,12 +112,31 @@ async fn test_produce_consume_seek() { // Seeking should allow replaying messages and skipping messages. #[tokio::test] async fn test_produce_consume_seek_partitions() { - let _r = env_logger::try_init(); + init_test_logger(); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_produce_consume_seek_partitions"); - populate_topic(&topic_name, 30, &value_fn, &key_fn, None, None).await; + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + produce_messages(&producer, &topic_name, 30, None, None).await; - let consumer = create_base_consumer(&rand_test_group(), None); + let group_id = rand_test_group(); + let consumer = + utils::consumer::create_base_consumer(&kafka_context.bootstrap_servers, &group_id, None) + .expect("could not create base consumer"); consumer.subscribe(&[topic_name.as_str()]).unwrap(); let mut partition_offset_map = HashMap::new(); @@ -156,12 +192,33 @@ async fn test_produce_consume_seek_partitions() { // All produced messages should be consumed. #[tokio::test] async fn test_produce_consume_iter() { - let _r = env_logger::try_init(); + init_test_logger(); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let start_time = current_time_millis(); let topic_name = rand_test_topic("test_produce_consume_iter"); - let message_map = populate_topic(&topic_name, 100, &value_fn, &key_fn, None, None).await; - let consumer = create_base_consumer(&rand_test_group(), None); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + let message_map = produce_messages(&producer, &topic_name, 100, None, None).await; + + let group_id = rand_test_group(); + let consumer = + utils::consumer::create_base_consumer(&kafka_context.bootstrap_servers, &group_id, None) + .expect("could not create base consumer"); consumer.subscribe(&[topic_name.as_str()]).unwrap(); for message in consumer.iter().take(100) { @@ -195,20 +252,30 @@ async fn test_pause_resume_consumer_iter() { const MESSAGE_COUNT: i32 = 300; const MESSAGES_PER_PAUSE: i32 = MESSAGE_COUNT / PAUSE_COUNT; - let _r = env_logger::try_init(); + init_test_logger(); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_pause_resume_consumer_iter"); - populate_topic( - &topic_name, - MESSAGE_COUNT, - &value_fn, - &key_fn, - Some(0), - None, - ) - .await; + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + produce_messages_to_partition(&producer, &topic_name, MESSAGE_COUNT as usize, 0).await; let group_id = rand_test_group(); - let consumer = create_base_consumer(&group_id, None); + let consumer = + utils::consumer::create_base_consumer(&kafka_context.bootstrap_servers, &group_id, None) + .expect("could not create base consumer"); consumer.subscribe(&[topic_name.as_str()]).unwrap(); for _ in 0..PAUSE_COUNT { @@ -236,17 +303,41 @@ async fn test_pause_resume_consumer_iter() { #[tokio::test] async fn test_consume_partition_order() { - let _r = env_logger::try_init(); + init_test_logger(); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_consume_partition_order"); - populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(0), None).await; - populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(1), None).await; - populate_topic(&topic_name, 4, &value_fn, &key_fn, Some(2), None).await; + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + produce_messages_to_partition(&producer, &topic_name, 4, 0).await; + produce_messages_to_partition(&producer, &topic_name, 4, 1).await; + produce_messages_to_partition(&producer, &topic_name, 4, 2).await; // Using partition queues should allow us to consume the partitions // in a round-robin fashion. { - let consumer = Arc::new(create_base_consumer(&rand_test_group(), None)); + let group_id = rand_test_group(); + let consumer = Arc::new( + utils::consumer::create_base_consumer( + &kafka_context.bootstrap_servers, + &group_id, + None, + ) + .expect("could not create base consumer"), + ); let mut tpl = TopicPartitionList::new(); tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) .unwrap(); @@ -274,7 +365,15 @@ async fn test_consume_partition_order() { // When not all partitions have been split into separate queues, the // unsplit partitions should still be accessible via the main queue. { - let consumer = Arc::new(create_base_consumer(&rand_test_group(), None)); + let group_id = rand_test_group(); + let consumer = Arc::new( + utils::consumer::create_base_consumer( + &kafka_context.bootstrap_servers, + &group_id, + None, + ) + .expect("could not create base consumer"), + ); let mut tpl = TopicPartitionList::new(); tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) .unwrap(); @@ -334,7 +433,15 @@ async fn test_consume_partition_order() { // should be continuously polled to serve callbacks, but it should not panic // or result in memory unsafety, etc. { - let consumer = Arc::new(create_base_consumer(&rand_test_group(), None)); + let group_id = rand_test_group(); + let consumer = Arc::new( + utils::consumer::create_base_consumer( + &kafka_context.bootstrap_servers, + &group_id, + None, + ) + .expect("could not create base consumer"), + ); let mut tpl = TopicPartitionList::new(); tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) .unwrap(); @@ -356,15 +463,28 @@ async fn test_consume_partition_order() { #[tokio::test] async fn test_produce_consume_message_queue_nonempty_callback() { - let _r = env_logger::try_init(); + init_test_logger(); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_produce_consume_message_queue_nonempty_callback"); - create_topic(&topic_name, 1).await; + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); - let consumer: BaseConsumer<_> = consumer_config(&rand_test_group(), None) - .create_with_context(ConsumerTestContext { _n: 64 }) - .expect("Consumer creation failed"); + let group_id = rand_test_group(); + let consumer = + utils::consumer::create_base_consumer(&kafka_context.bootstrap_servers, &group_id, None) + .expect("could not create base consumer"); let consumer = Arc::new(consumer); let mut tpl = TopicPartitionList::new(); @@ -410,7 +530,10 @@ async fn test_produce_consume_message_queue_nonempty_callback() { assert!(queue.poll(Duration::from_secs(0)).is_none()); // Populate the topic, and expect a wakeup notifying us of the new messages. - populate_topic(&topic_name, 2, &value_fn, &key_fn, None, None).await; + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + produce_messages(&producer, &topic_name, 2, None, None).await; wait_for_wakeups(1); // Read one of the messages. @@ -418,7 +541,7 @@ async fn test_produce_consume_message_queue_nonempty_callback() { // Add more messages to the topic. Expect no additional wakeups, as the // queue is not fully drained, for 1s. - populate_topic(&topic_name, 2, &value_fn, &key_fn, None, None).await; + produce_messages(&producer, &topic_name, 2, None, None).await; thread::sleep(Duration::from_secs(1)); assert_eq!(wakeups.load(Ordering::SeqCst), 1); @@ -432,7 +555,7 @@ async fn test_produce_consume_message_queue_nonempty_callback() { assert_eq!(wakeups.load(Ordering::SeqCst), 1); // Add another message, and expect a wakeup. - populate_topic(&topic_name, 1, &value_fn, &key_fn, None, None).await; + produce_messages(&producer, &topic_name, 1, None, None).await; wait_for_wakeups(2); // Expect no additional wakeups for 1s. @@ -441,7 +564,7 @@ async fn test_produce_consume_message_queue_nonempty_callback() { // Disable the queue and add another message. queue.set_nonempty_callback(|| ()); - populate_topic(&topic_name, 1, &value_fn, &key_fn, None, None).await; + produce_messages(&producer, &topic_name, 1, None, None).await; // Expect no additional wakeups for 1s. thread::sleep(Duration::from_secs(1)); diff --git a/tests/test_low_producers.rs b/tests/base_producer.rs similarity index 66% rename from tests/test_low_producers.rs rename to tests/base_producer.rs index c46c99681..c9e9061b5 100644 --- a/tests/test_low_producers.rs +++ b/tests/base_producer.rs @@ -1,6 +1,6 @@ //! Test data production using low level producers. -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::error::Error; use std::ffi::CString; use std::sync::Arc; @@ -8,9 +8,7 @@ use std::sync::Mutex; use std::thread; use std::time::Duration; -use maplit::hashmap; - -use rdkafka::config::ClientConfig; +use rdkafka::admin::AdminOptions; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; use rdkafka::message::{Header, Headers, Message, OwnedHeaders, OwnedMessage}; use rdkafka::producer::{ @@ -21,9 +19,11 @@ use rdkafka::types::RDKafkaRespErr; use rdkafka::util::current_time_millis; use rdkafka::{ClientContext, Statistics}; +use crate::utils::admin; +use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; +use crate::utils::producer::base_producer as base_producer_utils; use crate::utils::rand::*; -use crate::utils::*; mod utils; @@ -144,59 +144,33 @@ impl Partitioner for PanicPartitioner { } } -fn default_config(config_overrides: HashMap<&str, &str>) -> ClientConfig { - let mut config = ClientConfig::new(); - config - .set("bootstrap.servers", get_bootstrap_server()) - .set("message.timeout.ms", "5000"); - - for (key, value) in config_overrides { - config.set(key, value); - } - config -} - -fn base_producer(config_overrides: HashMap<&str, &str>) -> BaseProducer { - base_producer_with_context(PrintingContext { _n: 123 }, config_overrides) -} - -fn base_producer_with_context>( - context: C, - config_overrides: HashMap<&str, &str>, -) -> BaseProducer { - init_test_logger(); - default_config(config_overrides) - .create_with_context::>(context) - .unwrap() -} - -#[allow(dead_code)] -fn threaded_producer( - config_overrides: HashMap<&str, &str>, -) -> ThreadedProducer { - threaded_producer_with_context(PrintingContext { _n: 123 }, config_overrides) -} +// TESTS -fn threaded_producer_with_context( - context: C, - config_overrides: HashMap<&str, &str>, -) -> ThreadedProducer -where - Part: Partitioner + Send + Sync + 'static, - C: ProducerContext, -{ +#[tokio::test(flavor = "multi_thread")] +async fn test_base_producer_queue_full() { init_test_logger(); - default_config(config_overrides) - .create_with_context::>(context) - .unwrap() -} -// TESTS - -#[test] -fn test_base_producer_queue_full() { - let producer = base_producer(hashmap! { "queue.buffering.max.messages" => "10" }); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_base_producer_queue_full"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + PrintingContext { _n: 123 }, + &[("queue.buffering.max.messages", "10")], + ) + .expect("failed to create base producer"); let results = (0..30) .map(|id| { @@ -232,19 +206,34 @@ fn test_base_producer_queue_full() { assert_eq!(errors, 20); } -#[test] -fn test_base_producer_timeout() { +#[tokio::test(flavor = "multi_thread")] +async fn test_base_producer_timeout() { + init_test_logger(); + let context = CollectingContext::new(); - let bootstrap_server = get_bootstrap_server(); - let producer = base_producer_with_context( - context.clone(), - hashmap! { - "message.timeout.ms" => "100", - "bootstrap.servers" => &bootstrap_server, - }, - ); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_base_producer_timeout"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + context.clone(), + &[("message.timeout.ms", "100")], + ) + .expect("failed to create base producer"); + let results_count = (0..10) .map(|id| { producer.send( @@ -347,14 +336,35 @@ impl ProducerContext for HeaderCheckContext { } } -#[test] -fn test_base_producer_headers() { +#[tokio::test(flavor = "multi_thread")] +async fn test_base_producer_headers() { + init_test_logger(); + let ids_set = Arc::new(Mutex::new(HashSet::new())); let context = HeaderCheckContext { ids: ids_set.clone(), }; - let producer = base_producer_with_context(context, HashMap::new()); let topic_name = rand_test_topic("test_base_producer_headers"); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + context, + &[], + ) + .expect("failed to create base producer"); let results_count = (0..10) .map(|id| { @@ -391,11 +401,32 @@ fn test_base_producer_headers() { assert_eq!((*ids_set.lock().unwrap()).len(), 10); } -#[test] -fn test_threaded_producer_send() { +#[tokio::test(flavor = "multi_thread")] +async fn test_threaded_producer_send() { + init_test_logger(); + let context = CollectingContext::new(); - let producer = threaded_producer_with_context(context.clone(), HashMap::new()); let topic_name = rand_test_topic("test_threaded_producer_send"); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = base_producer_utils::create_threaded_producer_with_context( + &kafka_context.bootstrap_servers, + context.clone(), + &[], + ) + .expect("failed to create threaded producer"); let results_count = (0..10) .map(|id| { @@ -421,8 +452,10 @@ fn test_threaded_producer_send() { } } -#[test] -fn test_base_producer_opaque_arc() -> Result<(), Box> { +#[tokio::test(flavor = "multi_thread")] +async fn test_base_producer_opaque_arc() -> Result<(), Box> { + init_test_logger(); + struct OpaqueArcContext {} impl ClientContext for OpaqueArcContext {} impl ProducerContext for OpaqueArcContext { @@ -436,8 +469,27 @@ fn test_base_producer_opaque_arc() -> Result<(), Box> { let shared_count = Arc::new(Mutex::new(0)); let context = OpaqueArcContext {}; - let producer = base_producer_with_context(context, HashMap::new()); let topic_name = rand_test_topic("test_base_producer_opaque_arc"); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + context, + &[], + ) + .expect("failed to create base producer"); let results_count = (0..10) .map(|_| { @@ -454,9 +506,19 @@ fn test_base_producer_opaque_arc() -> Result<(), Box> { Ok(()) } -#[test] -fn test_fatal_errors() { - let producer = base_producer(HashMap::new()); +#[tokio::test(flavor = "multi_thread")] +async fn test_fatal_errors() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + PrintingContext { _n: 123 }, + &[], + ) + .expect("failed to create base producer"); assert_eq!(producer.client().fatal_error(), None); @@ -478,23 +540,38 @@ fn test_fatal_errors() { ) } -#[test] -fn test_register_custom_partitioner_linger_non_zero_key_null() { +#[tokio::test(flavor = "multi_thread")] +async fn test_register_custom_partitioner_linger_non_zero_key_null() { // Custom partitioner is not used when sticky.partitioning.linger.ms > 0 and key is null. // https://github.com/confluentinc/librdkafka/blob/081fd972fa97f88a1e6d9a69fc893865ffbb561a/src/rdkafka_msg.c#L1192-L1196 + init_test_logger(); + let context = CollectingContext::new_with_custom_partitioner(PanicPartitioner {}); - let mut config_overrides = HashMap::new(); - config_overrides.insert("sticky.partitioning.linger.ms", "10"); - let producer = base_producer_with_context(context.clone(), config_overrides); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_register_custom_partitioner_linger_non_zero_key_null"); + + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + context.clone(), + &[("sticky.partitioning.linger.ms", "10")], + ) + .expect("failed to create base producer"); producer - .send( - BaseRecord::<(), str, usize>::with_opaque_to( - &rand_test_topic("test_register_custom_partitioner_linger_non_zero_key_null"), - 0, - ) - .payload(""), - ) + .send(BaseRecord::<(), str, usize>::with_opaque_to(&topic_name, 0).payload("")) .unwrap(); producer.flush(Duration::from_secs(10)).unwrap(); @@ -507,11 +584,32 @@ fn test_register_custom_partitioner_linger_non_zero_key_null() { } } -#[test] -fn test_custom_partitioner_base_producer() { +#[tokio::test(flavor = "multi_thread")] +async fn test_custom_partitioner_base_producer() { + init_test_logger(); + let context = CollectingContext::new_with_custom_partitioner(FixedPartitioner::new(2)); - let producer = base_producer_with_context(context.clone(), HashMap::new()); let topic_name = rand_test_topic("test_custom_partitioner_base_producer"); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + context.clone(), + &[], + ) + .expect("failed to create base producer"); let results_count = (0..10) .map(|id| { @@ -535,11 +633,32 @@ fn test_custom_partitioner_base_producer() { } } -#[test] -fn test_custom_partitioner_threaded_producer() { +#[tokio::test(flavor = "multi_thread")] +async fn test_custom_partitioner_threaded_producer() { + init_test_logger(); + let context = CollectingContext::new_with_custom_partitioner(FixedPartitioner::new(2)); - let producer = threaded_producer_with_context(context.clone(), HashMap::new()); let topic_name = rand_test_topic("test_custom_partitioner_threaded_producer"); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = base_producer_utils::create_threaded_producer_with_context( + &kafka_context.bootstrap_servers, + context.clone(), + &[], + ) + .expect("failed to create threaded producer"); let results_count = (0..10) .map(|id| { diff --git a/tests/test_metadata.rs b/tests/metadata.rs similarity index 59% rename from tests/test_metadata.rs rename to tests/metadata.rs index 29449f2cb..242755dfa 100644 --- a/tests/test_metadata.rs +++ b/tests/metadata.rs @@ -2,6 +2,7 @@ use std::time::Duration; +use rdkafka::admin::AdminOptions; use rdkafka::config::ClientConfig; use rdkafka::consumer::{Consumer, StreamConsumer}; use rdkafka::error::KafkaError; @@ -9,33 +10,57 @@ use rdkafka::topic_partition_list::TopicPartitionList; use rdkafka_sys::types::RDKafkaConfRes; +use crate::utils::admin; +use crate::utils::containers::KafkaContext; +use crate::utils::logging::init_test_logger; +use crate::utils::producer; use crate::utils::rand::*; use crate::utils::*; mod utils; -fn create_consumer(group_id: &str) -> StreamConsumer { - ClientConfig::new() - .set("group.id", group_id) - .set("enable.partition.eof", "true") - .set("client.id", "rdkafka_integration_test_client") - .set("bootstrap.servers", get_bootstrap_server().as_str()) - .set("session.timeout.ms", "6000") - .set("debug", "all") - .set("auto.offset.reset", "earliest") - .create() - .expect("Failed to create StreamConsumer") +async fn create_consumer(kafka_context: &KafkaContext, group_id: &str) -> StreamConsumer { + utils::consumer::stream_consumer::create_stream_consumer_with_options( + &kafka_context.bootstrap_servers, + group_id, + &[ + ("enable.partition.eof", "true"), + ("client.id", "rdkafka_integration_test_client"), + ("session.timeout.ms", "6000"), + ("debug", "all"), + ], + ) + .await + .expect("Failed to create StreamConsumer") } #[tokio::test] async fn test_metadata() { - let _r = env_logger::try_init(); + init_test_logger(); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_metadata"); - populate_topic(&topic_name, 1, &value_fn, &key_fn, Some(0), None).await; - populate_topic(&topic_name, 1, &value_fn, &key_fn, Some(1), None).await; - populate_topic(&topic_name, 1, &value_fn, &key_fn, Some(2), None).await; - let consumer = create_consumer(&rand_test_group()); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + produce_messages_to_partition(&producer, &topic_name, 1, 0).await; + produce_messages_to_partition(&producer, &topic_name, 1, 1).await; + produce_messages_to_partition(&producer, &topic_name, 1, 2).await; + + let consumer = create_consumer(&kafka_context, &rand_test_group()).await; let metadata = consumer .fetch_metadata(None, Duration::from_secs(5)) @@ -90,11 +115,27 @@ async fn test_metadata() { #[tokio::test] async fn test_subscription() { - let _r = env_logger::try_init(); + init_test_logger(); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_subscription"); - populate_topic(&topic_name, 10, &value_fn, &key_fn, None, None).await; - let consumer = create_consumer(&rand_test_group()); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + produce_messages(&producer, &topic_name, 10, None, None).await; + let consumer = create_consumer(&kafka_context, &rand_test_group()).await; consumer.subscribe(&[topic_name.as_str()]).unwrap(); // Make sure the consumer joins the group. @@ -107,14 +148,32 @@ async fn test_subscription() { #[tokio::test] async fn test_group_membership() { - let _r = env_logger::try_init(); + init_test_logger(); + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let topic_name = rand_test_topic("test_group_membership"); let group_name = rand_test_group(); - populate_topic(&topic_name, 1, &value_fn, &key_fn, Some(0), None).await; - populate_topic(&topic_name, 1, &value_fn, &key_fn, Some(1), None).await; - populate_topic(&topic_name, 1, &value_fn, &key_fn, Some(2), None).await; - let consumer = create_consumer(&group_name); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(3)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + produce_messages_to_partition(&producer, &topic_name, 1, 0).await; + produce_messages_to_partition(&producer, &topic_name, 1, 1).await; + produce_messages_to_partition(&producer, &topic_name, 1, 2).await; + + let consumer = create_consumer(&kafka_context, &group_name).await; consumer.subscribe(&[topic_name.as_str()]).unwrap(); // Make sure the consumer joins the group. diff --git a/tests/test_topic_partition_list.rs b/tests/topic_partition_lists.rs similarity index 92% rename from tests/test_topic_partition_list.rs rename to tests/topic_partition_lists.rs index 2f0ac3511..69279b89f 100644 --- a/tests/test_topic_partition_list.rs +++ b/tests/topic_partition_lists.rs @@ -1,4 +1,4 @@ -use rdkafka::{Offset, TopicPartitionList}; +use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; /// Test topic partition list API and wrappers. diff --git a/tests/test_transactions.rs b/tests/transactions.rs similarity index 55% rename from tests/test_transactions.rs rename to tests/transactions.rs index 90857f585..9968c3251 100644 --- a/tests/test_transactions.rs +++ b/tests/transactions.rs @@ -1,12 +1,11 @@ //! Test transactions using the base consumer and producer. -use std::collections::HashMap; use std::error::Error; use std::time::Duration; use log::info; -use maplit::hashmap; +use rdkafka::admin::AdminOptions; use rdkafka::config::ClientConfig; use rdkafka::config::RDKafkaLogLevel; use rdkafka::consumer::{BaseConsumer, CommitMode, Consumer}; @@ -15,24 +14,45 @@ use rdkafka::producer::{BaseProducer, BaseRecord, Producer}; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::Timeout; +use crate::utils::admin; +use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; +use crate::utils::producer; use crate::utils::rand::*; -use utils::*; +use crate::utils::*; mod utils; -fn create_consumer( - config_overrides: Option>, -) -> Result { +async fn create_consumer( + kafka_context: &KafkaContext, + config_overrides: Option<&[(&str, &str)]>, +) -> Result, KafkaError> { init_test_logger(); - consumer_config(&rand_test_group(), config_overrides).create() + let group_id = rand_test_group(); + let mut config = ClientConfig::new(); + config + .set("group.id", &group_id) + .set("enable.partition.eof", "false") + .set("client.id", "rdkafka_integration_test_client") + .set("bootstrap.servers", &kafka_context.bootstrap_servers) + .set("session.timeout.ms", "6000") + .set("debug", "all") + .set("auto.offset.reset", "earliest"); + + if let Some(overrides) = config_overrides { + for (key, value) in overrides { + config.set(*key, *value); + } + } + + config.create_with_context(ConsumerTestContext { _n: 64 }) } -fn create_producer() -> Result { +fn create_producer(kafka_context: &KafkaContext) -> Result { init_test_logger(); let mut config = ClientConfig::new(); config - .set("bootstrap.servers", get_bootstrap_server()) + .set("bootstrap.servers", &kafka_context.bootstrap_servers) .set("message.timeout.ms", "5000") .set("enable.idempotence", "true") .set("transactional.id", rand_test_transactional_id()) @@ -46,14 +66,22 @@ enum IsolationLevel { ReadCommitted, } -fn count_records(topic: &str, iso: IsolationLevel) -> Result { - let consumer = create_consumer(Some(hashmap! { - "isolation.level" => match iso { - IsolationLevel::ReadUncommitted => "read_uncommitted", - IsolationLevel::ReadCommitted => "read_committed", - }, - "enable.partition.eof" => "true" - }))?; +async fn count_records( + kafka_context: &KafkaContext, + topic: &str, + iso: IsolationLevel, +) -> Result { + let isolation = match iso { + IsolationLevel::ReadUncommitted => "read_uncommitted", + IsolationLevel::ReadCommitted => "read_committed", + }; + + let consumer = create_consumer( + kafka_context, + Some(&[("isolation.level", isolation), ("enable.partition.eof", "true")]), + ) + .await?; + let mut tpl = TopicPartitionList::new(); tpl.add_partition(topic, 0); consumer.assign(&tpl)?; @@ -70,13 +98,39 @@ fn count_records(topic: &str, iso: IsolationLevel) -> Result #[tokio::test] async fn test_transaction_abort() -> Result<(), Box> { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let consume_topic = rand_test_topic("test_transaction_abort"); let produce_topic = rand_test_topic("test_transaction_abort"); - populate_topic(&consume_topic, 30, &value_fn, &key_fn, Some(0), None).await; + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&consume_topic, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create consume topic"); + admin_client + .create_topics( + &admin::new_topic_vec(&produce_topic, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create produce topic"); + + let future_producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + let _ = produce_messages_to_partition(&future_producer, &consume_topic, 30, 0).await; // Create consumer and subscribe to `consume_topic`. - let consumer = create_consumer(None)?; + let consumer = create_consumer(&kafka_context, None).await?; consumer.subscribe(&[&consume_topic])?; consumer.poll(Timeout::Never).unwrap()?; @@ -86,7 +140,7 @@ async fn test_transaction_abort() -> Result<(), Box> { consumer.commit(&commit_tpl, CommitMode::Sync).unwrap(); // Create a producer and start a transaction. - let producer = create_producer()?; + let producer = create_producer(&kafka_context)?; producer.init_transactions(Timeout::Never)?; producer.begin_transaction()?; @@ -118,11 +172,11 @@ async fn test_transaction_abort() -> Result<(), Box> { // Check that no records were produced in read committed mode, but that // the records are visible in read uncommitted mode. assert_eq!( - count_records(&produce_topic, IsolationLevel::ReadCommitted)?, + count_records(&kafka_context, &produce_topic, IsolationLevel::ReadCommitted).await?, 0, ); assert_eq!( - count_records(&produce_topic, IsolationLevel::ReadUncommitted)?, + count_records(&kafka_context, &produce_topic, IsolationLevel::ReadUncommitted).await?, 10, ); @@ -141,13 +195,39 @@ async fn test_transaction_abort() -> Result<(), Box> { #[tokio::test] async fn test_transaction_commit() -> Result<(), Box> { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); let consume_topic = rand_test_topic("test_transaction_commit"); let produce_topic = rand_test_topic("test_transaction_commit"); - populate_topic(&consume_topic, 30, &value_fn, &key_fn, Some(0), None).await; + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("Could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&consume_topic, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create consume topic"); + admin_client + .create_topics( + &admin::new_topic_vec(&produce_topic, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create produce topic"); + + let future_producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); + let _ = produce_messages_to_partition(&future_producer, &consume_topic, 30, 0).await; // Create consumer and subscribe to `consume_topic`. - let consumer = create_consumer(None)?; + let consumer = create_consumer(&kafka_context, None).await?; consumer.subscribe(&[&consume_topic])?; consumer.poll(Timeout::Never).unwrap()?; @@ -157,7 +237,7 @@ async fn test_transaction_commit() -> Result<(), Box> { consumer.commit(&commit_tpl, CommitMode::Sync).unwrap(); // Create a producer and start a transaction. - let producer = create_producer()?; + let producer = create_producer(&kafka_context)?; producer.init_transactions(Timeout::Never)?; producer.begin_transaction()?; @@ -184,11 +264,11 @@ async fn test_transaction_commit() -> Result<(), Box> { // Check that 10 records were produced. assert_eq!( - count_records(&produce_topic, IsolationLevel::ReadUncommitted)?, + count_records(&kafka_context, &produce_topic, IsolationLevel::ReadUncommitted).await?, 10, ); assert_eq!( - count_records(&produce_topic, IsolationLevel::ReadCommitted)?, + count_records(&kafka_context, &produce_topic, IsolationLevel::ReadCommitted).await?, 10, ); diff --git a/tests/utils/consumer/mod.rs b/tests/utils/consumer/mod.rs index fa9e3e518..e2afbb0a3 100644 --- a/tests/utils/consumer/mod.rs +++ b/tests/utils/consumer/mod.rs @@ -1,6 +1,7 @@ pub mod stream_consumer; use crate::utils::rand::rand_test_group; +use crate::utils::ConsumerTestContext; use anyhow::{bail, Context}; use backon::{BlockingRetryable, ExponentialBuilder}; use rdkafka::config::FromClientConfig; @@ -44,6 +45,34 @@ pub async fn create_unsubscribed_base_consumer( BaseConsumer::from_config(&consumer_client_config).context("Failed to create consumer") } +pub fn create_base_consumer( + bootstrap_servers: &str, + consumer_group: &str, + config_overrides: Option<&[(&str, &str)]>, +) -> anyhow::Result> { + let mut consumer_client_config = ClientConfig::default(); + consumer_client_config.set("group.id", consumer_group); + consumer_client_config.set("client.id", "rdkafka_integration_test_client"); + consumer_client_config.set("bootstrap.servers", bootstrap_servers); + consumer_client_config.set("enable.partition.eof", "false"); + consumer_client_config.set("session.timeout.ms", "6000"); + consumer_client_config.set("enable.auto.commit", "false"); + consumer_client_config.set("debug", "all"); + consumer_client_config.set("auto.offset.reset", "earliest"); + + if let Some(overrides) = config_overrides { + for (key, value) in overrides { + consumer_client_config.set(*key, *value); + } + } + + consumer_client_config + .create_with_context::>( + ConsumerTestContext { _n: 64 }, + ) + .context("Failed to create consumer with context") +} + pub async fn poll_x_times_for_messages( consumer: &BaseConsumer, times_to_poll: i32, diff --git a/tests/utils/producer/base_producer.rs b/tests/utils/producer/base_producer.rs index 6f8bfe05a..0ad7c4fac 100644 --- a/tests/utils/producer/base_producer.rs +++ b/tests/utils/producer/base_producer.rs @@ -1,6 +1,8 @@ use anyhow::{bail, Context}; -use rdkafka::config::FromClientConfig; -use rdkafka::producer::{BaseProducer, BaseRecord, Producer}; +use rdkafka::config::{FromClientConfig, FromClientConfigAndContext}; +use rdkafka::producer::{ + BaseProducer, BaseRecord, Partitioner, Producer, ProducerContext, ThreadedProducer, +}; use rdkafka::util::Timeout; use rdkafka::ClientConfig; use std::time::Duration; @@ -29,6 +31,48 @@ pub fn create_base_producer(config: &ClientConfig) -> anyhow::Result( + bootstrap_servers: &str, + context: C, + config_overrides: &[(&str, &str)], +) -> anyhow::Result> +where + C: ProducerContext, + Part: Partitioner, +{ + let mut producer_client_config = ClientConfig::default(); + producer_client_config.set("bootstrap.servers", bootstrap_servers); + producer_client_config.set("message.timeout.ms", "5000"); + + for (key, value) in config_overrides { + producer_client_config.set(*key, *value); + } + + BaseProducer::from_config_and_context(&producer_client_config, context) + .context("error creating base producer with context") +} + +pub fn create_threaded_producer_with_context( + bootstrap_servers: &str, + context: C, + config_overrides: &[(&str, &str)], +) -> anyhow::Result> +where + C: ProducerContext, + Part: Partitioner + Send + Sync + 'static, +{ + let mut producer_client_config = ClientConfig::default(); + producer_client_config.set("bootstrap.servers", bootstrap_servers); + producer_client_config.set("message.timeout.ms", "5000"); + + for (key, value) in config_overrides { + producer_client_config.set(*key, *value); + } + + ThreadedProducer::from_config_and_context(&producer_client_config, context) + .context("error creating threaded producer with context") +} + pub async fn send_record( producer: &BaseProducer, record: BaseRecord<'_, [u8; 4], str>, From ddfb0a030af69f5a63a9bcb9fe6d2225e79e1cc6 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 21 May 2026 17:30:06 -0500 Subject: [PATCH 27/56] test: clean up clippy findings under -Dwarnings cargo clippy --tests -- -Dwarnings is part of the lint CI job. Replace is_err()+unwrap_err() with if let Err, drop expect(&format!()) in favour of unwrap_or_else, prefer first() over get(0), and remove unused imports across the new testcontainers-based test suite. --- tests/admin.rs | 34 +++++++++---------- tests/producer.rs | 18 ++++------ tests/stream_consumers.rs | 2 +- tests/transactions.rs | 47 ++++++++++++++++++++------- tests/utils/admin.rs | 8 ++--- tests/utils/producer/base_producer.rs | 7 ++-- 6 files changed, 66 insertions(+), 50 deletions(-) diff --git a/tests/admin.rs b/tests/admin.rs index fde2cbf58..932a33a98 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -1,12 +1,12 @@ use crate::utils::admin::create_topic; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; -use crate::utils::rand::{rand_test_group, rand_test_topic}; +use crate::utils::rand::rand_test_topic; use crate::utils::{get_broker_version, KafkaVersion}; use backon::{BlockingRetryable, ExponentialBuilder}; use rdkafka::admin::{ - AdminClient, AdminOptions, AlterConfig, ConfigEntry, ConfigSource, GroupResult, NewPartitions, - NewTopic, OwnedResourceSpecifier, ResourceSpecifier, TopicReplication, + AdminClient, AdminOptions, AlterConfig, ConfigEntry, ConfigSource, NewPartitions, NewTopic, + OwnedResourceSpecifier, ResourceSpecifier, TopicReplication, }; use rdkafka::client::DefaultClientContext; use rdkafka::error::KafkaError; @@ -42,12 +42,8 @@ pub async fn test_topic_creation() { ); }; - let create_topic_result = create_topic(&admin_client, &test_topic_name).await; - if create_topic_result.is_err() { - panic!( - "could not create topic: {}", - create_topic_result.unwrap_err() - ); + if let Err(err) = create_topic(&admin_client, &test_topic_name).await { + panic!("could not create topic: {}", err); }; } @@ -101,9 +97,9 @@ pub async fn test_topic_create_and_delete() { // Verify metadata let metadata1 = utils::consumer::fetch_consumer_metadata(&consumer_client, &topic_name_1) - .expect(&format!("failed to fetch metadata for {}", &topic_name_1)); + .unwrap_or_else(|_| panic!("failed to fetch metadata for {}", &topic_name_1)); let metadata2 = utils::consumer::fetch_consumer_metadata(&consumer_client, &topic_name_2) - .expect(&format!("failed to fetch metadata for {}", topic_name_2)); + .unwrap_or_else(|_| panic!("failed to fetch metadata for {}", topic_name_2)); assert_eq!(1, metadata1.topics().len()); assert_eq!(1, metadata2.topics().len()); let metadata_topic1 = &metadata1.topics()[0]; @@ -126,10 +122,10 @@ pub async fn test_topic_create_and_delete() { .expect("could not describe configs"); let topic_config1 = &config_resource_results[0] .as_ref() - .expect(&format!("failed to describe config for {}", &topic_name_1)); + .unwrap_or_else(|_| panic!("failed to describe config for {}", &topic_name_1)); let topic_config2 = &config_resource_results[1] .as_ref() - .expect(&format!("failed to describe config for {}", &topic_name_2)); + .unwrap_or_else(|_| panic!("failed to describe config for {}", &topic_name_2)); let mut expected_entry1 = ConfigEntry { name: "max.message.bytes".into(), value: Some("1234".into()), @@ -185,7 +181,7 @@ pub async fn test_topic_create_and_delete() { let mut tries = 0; loop { let metadata = utils::consumer::fetch_consumer_metadata(&consumer_client, &topic_name_1) - .expect(&format!("failed to fetch metadata for {}", &topic_name_1)); + .unwrap_or_else(|_| panic!("failed to fetch metadata for {}", &topic_name_1)); let topic = &metadata.topics()[0]; let n = topic.partitions().len(); if n == 5 { @@ -204,9 +200,9 @@ pub async fn test_topic_create_and_delete() { .expect("topic deletion failed"); assert_eq!(res, &[Ok(topic_name_1.clone()), Ok(topic_name_2.clone())]); utils::consumer::verify_topic_deleted(&consumer_client, &topic_name_1) - .expect(&format!("could not delete topic for {}", &topic_name_1)); + .unwrap_or_else(|_| panic!("could not delete topic for {}", &topic_name_1)); utils::consumer::verify_topic_deleted(&consumer_client, &topic_name_2) - .expect(&format!("could not delete topic for {}", &topic_name_2)); + .unwrap_or_else(|_| panic!("could not delete topic for {}", &topic_name_2)); } /// Verify that incorrect replication configurations are ignored when @@ -378,7 +374,7 @@ pub async fn test_mixed_success_results() { .expect("topic creation failed"); assert_eq!(res, &[Ok(name1.clone())]); let _ = utils::consumer::fetch_consumer_metadata(&consumer_client, &name1) - .expect(&format!("could not fetch consumer metadata for {}", name1)); + .unwrap_or_else(|_| panic!("could not fetch consumer metadata for {}", name1)); let res = admin_client .create_topics(vec![&topic1, &topic2], &opts) @@ -392,7 +388,7 @@ pub async fn test_mixed_success_results() { ] ); let _ = utils::consumer::fetch_consumer_metadata(&consumer_client, &name2) - .expect(&format!("could not fetch consumer metadata for {}", name2)); + .unwrap_or_else(|_| panic!("could not fetch consumer metadata for {}", name2)); let res = admin_client .delete_topics(&[&name1], &opts) @@ -400,7 +396,7 @@ pub async fn test_mixed_success_results() { .expect("topic deletion failed"); assert_eq!(res, &[Ok(name1.clone())]); utils::consumer::verify_topic_deleted(&consumer_client, &name1) - .expect(&format!("could not verify topic \"{}\" was deleted", name1)); + .unwrap_or_else(|_| panic!("could not verify topic \"{}\" was deleted", name1)); let res = admin_client .delete_topics(&[&name2, &name1], &opts) diff --git a/tests/producer.rs b/tests/producer.rs index b41a9d394..f3d53feff 100644 --- a/tests/producer.rs +++ b/tests/producer.rs @@ -30,12 +30,8 @@ pub async fn test_basic_produce() { admin_client_result.unwrap_err() ); }; - let create_topic_result = create_topic(&admin_client, &test_topic_name).await; - if create_topic_result.is_err() { - panic!( - "could not create topic: {}", - create_topic_result.unwrap_err() - ); + if let Err(err) = create_topic(&admin_client, &test_topic_name).await { + panic!("could not create topic: {}", err); } let consumer_result = @@ -59,10 +55,10 @@ pub async fn test_basic_produce() { let record = BaseRecord::to(&test_topic_name) // destination topic .key(&[1, 2, 3, 4]) // message key .payload("content"); // message payload - let send_record_result = - crate::utils::producer::base_producer::send_record(&base_producer, record).await; - if send_record_result.is_err() { - panic!("could not send record: {}", send_record_result.unwrap_err()); + if let Err(err) = + crate::utils::producer::base_producer::send_record(&base_producer, record).await + { + panic!("could not send record: {}", err); } let messages_result = poll_x_times_for_messages(&consumer, 10).await; @@ -72,7 +68,7 @@ pub async fn test_basic_produce() { if messages.len() != 1 { panic!("expected exactly one message"); } - let borrowed_next_message = messages.get(0).unwrap(); + let borrowed_next_message = messages.first().unwrap(); let owned_next_message = borrowed_next_message.detach(); let Some(message_payload) = owned_next_message.payload() else { diff --git a/tests/stream_consumers.rs b/tests/stream_consumers.rs index c5030734e..8c40723f3 100644 --- a/tests/stream_consumers.rs +++ b/tests/stream_consumers.rs @@ -11,7 +11,7 @@ use rdkafka_sys::RDKafkaErrorCode; use tokio::time::{self, Duration}; use rdkafka::admin::AdminOptions; -use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer}; +use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer}; use rdkafka::error::KafkaError; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::current_time_millis; diff --git a/tests/transactions.rs b/tests/transactions.rs index 9968c3251..a3a0fe791 100644 --- a/tests/transactions.rs +++ b/tests/transactions.rs @@ -78,7 +78,10 @@ async fn count_records( let consumer = create_consumer( kafka_context, - Some(&[("isolation.level", isolation), ("enable.partition.eof", "true")]), + Some(&[ + ("isolation.level", isolation), + ("enable.partition.eof", "true"), + ]), ) .await?; @@ -124,9 +127,10 @@ async fn test_transaction_abort() -> Result<(), Box> { .await .expect("could not create produce topic"); - let future_producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) - .await - .expect("Could not create Future producer"); + let future_producer = + producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); let _ = produce_messages_to_partition(&future_producer, &consume_topic, 30, 0).await; // Create consumer and subscribe to `consume_topic`. @@ -172,11 +176,21 @@ async fn test_transaction_abort() -> Result<(), Box> { // Check that no records were produced in read committed mode, but that // the records are visible in read uncommitted mode. assert_eq!( - count_records(&kafka_context, &produce_topic, IsolationLevel::ReadCommitted).await?, + count_records( + &kafka_context, + &produce_topic, + IsolationLevel::ReadCommitted + ) + .await?, 0, ); assert_eq!( - count_records(&kafka_context, &produce_topic, IsolationLevel::ReadUncommitted).await?, + count_records( + &kafka_context, + &produce_topic, + IsolationLevel::ReadUncommitted + ) + .await?, 10, ); @@ -221,9 +235,10 @@ async fn test_transaction_commit() -> Result<(), Box> { .await .expect("could not create produce topic"); - let future_producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) - .await - .expect("Could not create Future producer"); + let future_producer = + producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("Could not create Future producer"); let _ = produce_messages_to_partition(&future_producer, &consume_topic, 30, 0).await; // Create consumer and subscribe to `consume_topic`. @@ -264,11 +279,21 @@ async fn test_transaction_commit() -> Result<(), Box> { // Check that 10 records were produced. assert_eq!( - count_records(&kafka_context, &produce_topic, IsolationLevel::ReadUncommitted).await?, + count_records( + &kafka_context, + &produce_topic, + IsolationLevel::ReadUncommitted + ) + .await?, 10, ); assert_eq!( - count_records(&kafka_context, &produce_topic, IsolationLevel::ReadCommitted).await?, + count_records( + &kafka_context, + &produce_topic, + IsolationLevel::ReadCommitted + ) + .await?, 10, ); diff --git a/tests/utils/admin.rs b/tests/utils/admin.rs index 2d9e0bb0e..e1fad1a4c 100644 --- a/tests/utils/admin.rs +++ b/tests/utils/admin.rs @@ -17,12 +17,12 @@ pub async fn create_topic( topic_name: &'_ str, ) -> anyhow::Result { let topic_results = admin_client - .create_topics(&new_topic_vec(&topic_name, None), &AdminOptions::default()) + .create_topics(&new_topic_vec(topic_name, None), &AdminOptions::default()) .await .context("error creating topics")?; for topic_result in topic_results { - if topic_result.is_err() { - bail!("failed to create topic: {:?}", topic_result.unwrap_err()); + if let Err(err) = topic_result { + bail!("failed to create topic: {:?}", err); }; } Ok(topic_name.to_string()) @@ -30,7 +30,7 @@ pub async fn create_topic( pub fn new_topic_vec(topic_name: &'_ str, num_partitions: Option) -> Vec> { let new_topic = NewTopic::new( - &topic_name, + topic_name, num_partitions.unwrap_or(1), TopicReplication::Fixed(1), ); diff --git a/tests/utils/producer/base_producer.rs b/tests/utils/producer/base_producer.rs index 0ad7c4fac..2b3af3890 100644 --- a/tests/utils/producer/base_producer.rs +++ b/tests/utils/producer/base_producer.rs @@ -77,11 +77,10 @@ pub async fn send_record( producer: &BaseProducer, record: BaseRecord<'_, [u8; 4], str>, ) -> anyhow::Result<()> { - let send_result = producer.send(record); - if send_result.is_err() { - bail!("could not produce record: {:?}", send_result.unwrap_err()); + if let Err(err) = producer.send(record) { + bail!("could not produce record: {:?}", err); } - if poll_and_flush(&producer).is_err() { + if poll_and_flush(producer).is_err() { bail!("could not poll and flush base producer") }; From dac1ee9754c6aa3d165c42707c64fd4a24ecbb20 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 21 May 2026 17:30:15 -0500 Subject: [PATCH 28/56] test: fix integration tests for the testcontainers Kafka image A handful of tests were still wired up for the Bitnami broker: - containers: set KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR and KAFKA_TRANSACTION_STATE_LOG_MIN_ISR to 1, otherwise the single-broker testcontainers image cannot create __transaction_state and the transaction tests hang. - base_producer::test_base_producer_timeout: drop the explicit topic creation so messages still time out under the 100ms deadline. The upstream test relied on producing to a non-existent topic; once the topic existed the broker delivered the messages in well under 100ms. - consumer_groups::test_delete_unknown_group: expect GroupIdNotFound, matching the upstream assertion and the broker's response after the consumer-groups topic is initialised. - metadata::test_metadata: assert against BROKER_ID (1, the testcontainers default) instead of the Bitnami-era 0, and drop the hardcoded 9092 port check since the container's host port is allocated dynamically. --- tests/base_producer.rs | 16 +++------------- tests/consumer_groups.rs | 2 +- tests/metadata.rs | 21 ++++++++++----------- tests/utils/containers.rs | 13 ++++++++++--- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/tests/base_producer.rs b/tests/base_producer.rs index c9e9061b5..2cc94673d 100644 --- a/tests/base_producer.rs +++ b/tests/base_producer.rs @@ -12,8 +12,7 @@ use rdkafka::admin::AdminOptions; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; use rdkafka::message::{Header, Headers, Message, OwnedHeaders, OwnedMessage}; use rdkafka::producer::{ - BaseProducer, BaseRecord, DeliveryResult, NoCustomPartitioner, Partitioner, Producer, - ProducerContext, ThreadedProducer, + BaseRecord, DeliveryResult, NoCustomPartitioner, Partitioner, Producer, ProducerContext, }; use rdkafka::types::RDKafkaRespErr; use rdkafka::util::current_time_millis; @@ -214,19 +213,10 @@ async fn test_base_producer_timeout() { let kafka_context = KafkaContext::shared() .await .expect("could not create kafka context"); + // Send to a topic that does not exist; messages cannot be routed within + // the 100ms timeout and should be reported as MessageTimedOut. let topic_name = rand_test_topic("test_base_producer_timeout"); - let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) - .await - .expect("Could not create admin client"); - admin_client - .create_topics( - &admin::new_topic_vec(&topic_name, Some(1)), - &AdminOptions::default(), - ) - .await - .expect("could not create topic"); - let producer = base_producer_utils::create_base_producer_with_context( &kafka_context.bootstrap_servers, context.clone(), diff --git a/tests/consumer_groups.rs b/tests/consumer_groups.rs index 71d44c8fd..7ac288551 100644 --- a/tests/consumer_groups.rs +++ b/tests/consumer_groups.rs @@ -75,7 +75,7 @@ pub async fn test_delete_unknown_group() { let res = admin_client .delete_groups(&[&unknown_group_name], &AdminOptions::default()) .await; - let expected: GroupResult = Err((unknown_group_name, RDKafkaErrorCode::NotCoordinator)); + let expected: GroupResult = Err((unknown_group_name, RDKafkaErrorCode::GroupIdNotFound)); assert_eq!(res, Ok(vec![expected])); } diff --git a/tests/metadata.rs b/tests/metadata.rs index 242755dfa..ba67cefcf 100644 --- a/tests/metadata.rs +++ b/tests/metadata.rs @@ -67,20 +67,19 @@ async fn test_metadata() { .unwrap(); let orig_broker_id = metadata.orig_broker_id(); // The orig_broker_id may be -1 if librdkafka's bootstrap "broker" handles - // the request. - if orig_broker_id != -1 && orig_broker_id != 0 { + // the request. The testcontainers Kafka image assigns BROKER_ID=1. + if orig_broker_id != -1 && orig_broker_id != BROKER_ID { panic!( - "metadata.orig_broker_id = {}, not 0 or 1 as expected", - orig_broker_id + "metadata.orig_broker_id = {}, not -1 or {} as expected", + orig_broker_id, BROKER_ID ) } assert!(!metadata.orig_broker_name().is_empty()); let broker_metadata = metadata.brokers(); assert_eq!(broker_metadata.len(), 1); - assert_eq!(broker_metadata[0].id(), 0); + assert_eq!(broker_metadata[0].id(), BROKER_ID); assert!(!broker_metadata[0].host().is_empty()); - assert_eq!(broker_metadata[0].port(), 9092); let topic_metadata = metadata .topics() @@ -101,11 +100,11 @@ async fn test_metadata() { assert_eq!(ids, vec![0, 1, 2]); assert_eq!(topic_metadata.error(), None); assert_eq!(topic_metadata.partitions().len(), 3); - assert_eq!(topic_metadata.partitions()[0].leader(), 0); - assert_eq!(topic_metadata.partitions()[1].leader(), 0); - assert_eq!(topic_metadata.partitions()[2].leader(), 0); - assert_eq!(topic_metadata.partitions()[0].replicas(), &[0]); - assert_eq!(topic_metadata.partitions()[0].isr(), &[0]); + assert_eq!(topic_metadata.partitions()[0].leader(), BROKER_ID); + assert_eq!(topic_metadata.partitions()[1].leader(), BROKER_ID); + assert_eq!(topic_metadata.partitions()[2].leader(), BROKER_ID); + assert_eq!(topic_metadata.partitions()[0].replicas(), &[BROKER_ID]); + assert_eq!(topic_metadata.partitions()[0].isr(), &[BROKER_ID]); let metadata_one_topic = consumer .fetch_metadata(Some(&topic_name), Duration::from_secs(5)) diff --git a/tests/utils/containers.rs b/tests/utils/containers.rs index 560d7ce7e..93682f2d6 100644 --- a/tests/utils/containers.rs +++ b/tests/utils/containers.rs @@ -4,9 +4,11 @@ use std::sync::Arc; use testcontainers_modules::kafka::apache::Kafka; use testcontainers_modules::testcontainers::core::ContainerPort; use testcontainers_modules::testcontainers::runners::AsyncRunner; -use testcontainers_modules::testcontainers::{ContainerAsync, Image}; +use testcontainers_modules::testcontainers::{ContainerAsync, Image, ImageExt}; use tokio::sync::OnceCell; +type KafkaImage = testcontainers_modules::testcontainers::core::ContainerRequest; + pub struct KafkaContext { kafka_node: ContainerAsync, pub bootstrap_servers: String, @@ -50,8 +52,13 @@ impl KafkaContext { } async fn init() -> anyhow::Result> { - let kafka_container = Kafka::default(); - let kafka_version = kafka_container.tag().to_string(); + let kafka_container: KafkaImage = Kafka::default() + // The single-broker testcontainers image needs replication and ISR + // overrides; otherwise transactions hang because __transaction_state + // can't reach its default replication factor of 3. + .with_env_var("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1") + .with_env_var("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1"); + let kafka_version = Kafka::default().tag().to_string(); let kafka_node = kafka_container .start() From 7361259564dec33cb80a0cd4a4b18f3c9ea7d54d Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 21 May 2026 17:42:51 -0500 Subject: [PATCH 29/56] test: pin testcontainers-modules to 0.12.1 for the CI rustc 1.85 floor testcontainers-modules 0.13 pulls testcontainers 0.25.x and a transitive tree (tonic 0.14, darling 0.23, home 0.5.12, serde_with 3.20, icu_* 2.2, ...) that has all jumped to MSRV 1.88. The library's CI pins rust_version: 1.85, so cargo refuses to resolve those packages even though the rdkafka crate itself stays well under 1.85. Downgrading the dev-dep to 0.12.1 keeps the Kafka::default() API we use and pulls testcontainers 0.24.x, whose tree is happy on 1.85. --- Cargo.lock | 576 +++++++++-------------------------------------------- Cargo.toml | 2 +- 2 files changed, 94 insertions(+), 484 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38894688f..89353adaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,7 +62,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -73,7 +73,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -82,22 +82,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "astral-tokio-tar" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec179a06c1769b1e42e1e2cbe74c7dcdb3d6383c838454d063eaac5bbb7ebbe5" -dependencies = [ - "filetime", - "futures-core", - "libc", - "portable-atomic", - "rustc-hash", - "tokio", - "tokio-stream", - "xattr", -] - [[package]] name = "async-attributes" version = "1.1.2" @@ -274,28 +258,6 @@ dependencies = [ "wasm-bindgen-futures", ] -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "async-task" version = "4.7.1" @@ -325,49 +287,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "axum" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" -dependencies = [ - "axum-core", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "backon" version = "1.6.0" @@ -395,10 +314,10 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools", "proc-macro2", "quote", "regex", @@ -407,6 +326,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.1" @@ -428,17 +353,13 @@ dependencies = [ [[package]] name = "bollard" -version = "0.19.4" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87a52479c9237eb04047ddb94788c41ca0d26eaff8b697ecfbb4c32f7fdc3b1b" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" dependencies = [ - "async-stream", "base64 0.22.1", - "bitflags", - "bollard-buildkit-proto", "bollard-stubs", "bytes", - "chrono", "futures-core", "futures-util", "hex", @@ -451,9 +372,7 @@ dependencies = [ "hyper-util", "hyperlocal", "log", - "num", "pin-project-lite", - "rand", "rustls", "rustls-native-certs", "rustls-pemfile", @@ -465,53 +384,23 @@ dependencies = [ "serde_urlencoded", "thiserror", "tokio", - "tokio-stream", "tokio-util", - "tonic", "tower-service", "url", "winapi", ] -[[package]] -name = "bollard-buildkit-proto" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" -dependencies = [ - "prost", - "prost-types", - "tonic", - "tonic-prost", - "ureq", -] - [[package]] name = "bollard-stubs" -version = "1.49.1-rc.28.4.0" +version = "1.47.1-rc.27.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5731fe885755e92beff1950774068e0cae67ea6ec7587381536fca84f1779623" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" dependencies = [ - "base64 0.22.1", - "bollard-buildkit-proto", - "bytes", - "chrono", - "prost", "serde", - "serde_json", "serde_repr", "serde_with", ] -[[package]] -name = "bs58" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" -dependencies = [ - "tinyvec", -] - [[package]] name = "bumpalo" version = "3.20.2" @@ -685,14 +574,14 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "darling" -version = "0.23.0" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ "darling_core", "darling_macro", @@ -700,10 +589,11 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.23.0" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ + "fnv", "ident_case", "proc-macro2", "quote", @@ -713,9 +603,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.23.0" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", @@ -814,7 +704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1044,25 +934,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "h2" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.14.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -1103,11 +974,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "home" -version = "0.5.12" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1165,7 +1036,6 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", "http", "http-body", "httparse", @@ -1207,19 +1077,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-timeout" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" -dependencies = [ - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.20" @@ -1281,13 +1138,12 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", "potential_utf", - "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1295,9 +1151,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "93cca704c2d63cf8a91f5c2c5f88e027940dede132319b85a52939db9758f7e5" dependencies = [ "displaydoc", "litemap", @@ -1308,10 +1164,11 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "8b24a59706036ba941c9476a55cd57b82b77f38a3c667d637ee7cabbc85eaedc" dependencies = [ + "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -1322,38 +1179,42 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "f5a97b8ac6235e69506e8dacfb2adf38461d2ce6d3e9bd9c94c4cbc3cd4400a4" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", + "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", "icu_locale_core", + "stable_deref_trait", + "tinystr", "writeable", "yoke", "zerofrom", @@ -1380,9 +1241,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -1426,15 +1287,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -1570,24 +1422,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1625,76 +1465,12 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1773,7 +1549,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1813,26 +1589,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "pin-project" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1933,38 +1689,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" -dependencies = [ - "anyhow", - "itertools 0.14.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "prost-types" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" -dependencies = [ - "prost", -] - [[package]] name = "quote" version = "1.0.45" @@ -2056,6 +1780,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "ref-cast" version = "1.0.25" @@ -2131,11 +1864,11 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2144,7 +1877,6 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ - "log", "once_cell", "ring", "rustls-pki-types", @@ -2258,7 +1990,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "core-foundation", "core-foundation-sys", "libc", @@ -2343,19 +2075,19 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" dependencies = [ "base64 0.22.1", - "bs58", "chrono", "hex", "indexmap 1.9.3", "indexmap 2.14.0", "schemars 0.9.0", "schemars 1.2.1", - "serde_core", + "serde", + "serde_derive", "serde_json", "serde_with_macros", "time", @@ -2363,9 +2095,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" dependencies = [ "darling", "proc-macro2", @@ -2463,7 +2195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2529,12 +2261,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" - [[package]] name = "synstructure" version = "0.13.2" @@ -2548,13 +2274,13 @@ dependencies = [ [[package]] name = "testcontainers" -version = "0.25.2" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3ac71069f20ecfa60c396316c283fbf35e6833a53dff551a31b5458da05edc" +checksum = "23bb7577dca13ad86a78e8271ef5d322f37229ec83b8d98da6d996c588a1ddb1" dependencies = [ - "astral-tokio-tar", "async-trait", "bollard", + "bollard-stubs", "bytes", "docker_credential", "either", @@ -2570,16 +2296,16 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", + "tokio-tar", "tokio-util", - "ulid", "url", ] [[package]] name = "testcontainers-modules" -version = "0.13.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1966329d5bb3f89d33602d2db2da971fb839f9297dad16527abf4564e2ae0a6d" +checksum = "eac95cde96549fc19c6bf19ef34cc42bd56e264c1cb97e700e21555be0ecf9e2" dependencies = [ "testcontainers", ] @@ -2642,24 +2368,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] -[[package]] -name = "tinyvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.52.3" @@ -2707,6 +2419,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -2750,71 +2477,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tonic" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" -dependencies = [ - "async-trait", - "axum", - "base64 0.22.1", - "bytes", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "socket2", - "sync_wrapper", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-prost" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" -dependencies = [ - "bytes", - "prost", - "tonic", -] - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 2.14.0", - "pin-project-lite", - "slab", - "sync_wrapper", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - [[package]] name = "tower-service" version = "0.3.3" @@ -2858,16 +2520,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "ulid" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" -dependencies = [ - "rand", - "web-time", -] - [[package]] name = "unicode-ident" version = "1.0.24" @@ -2880,52 +2532,18 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" -dependencies = [ - "base64 0.22.1", - "log", - "percent-encoding", - "rustls", - "rustls-pki-types", - "ureq-proto", - "utf8-zero", -] - -[[package]] -name = "ureq-proto" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" -dependencies = [ - "base64 0.22.1", - "http", - "httparse", - "log", -] - [[package]] name = "url" -version = "2.5.8" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", "percent-encoding", "serde", - "serde_derive", ] -[[package]] -name = "utf8-zero" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -3029,16 +2647,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "winapi" version = "0.3.9" @@ -3395,6 +3003,7 @@ dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] @@ -3403,6 +3012,7 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/Cargo.toml b/Cargo.toml index 2d843031a..e3f56330f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ rand = "0.9.1" regex = "1.11.1" smol = "2.0.2" tokio = { version = "1.18", features = ["macros", "rt-multi-thread", "time"] } -testcontainers-modules = { version = "0.13.0", features = ["kafka"] } +testcontainers-modules = { version = "0.12.1", features = ["kafka"] } anyhow = { version = "1.0.100" } # These features are re-exports of the features that the rdkafka-sys crate From 6b8d99a35f47d93a64c4f686f5ac6612f1f3a48f Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 21 May 2026 17:47:48 -0500 Subject: [PATCH 30/56] test: point base_producer_timeout at an unreachable broker Producing to a non-existent topic with message.timeout.ms=100 isn't a reliable trigger for MessageTimedOut on fast CI hardware: with the broker's auto.create.topics.enable default, the topic gets created and the message delivered within the timeout, and the test sees None instead of MessageTimedOut. Switching the producer's bootstrap.servers to 127.0.0.1:1 keeps the test focused on the delivery callback contract without depending on broker timing. --- tests/base_producer.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/base_producer.rs b/tests/base_producer.rs index 2cc94673d..541e5edc7 100644 --- a/tests/base_producer.rs +++ b/tests/base_producer.rs @@ -210,15 +210,15 @@ async fn test_base_producer_timeout() { init_test_logger(); let context = CollectingContext::new(); - let kafka_context = KafkaContext::shared() - .await - .expect("could not create kafka context"); - // Send to a topic that does not exist; messages cannot be routed within - // the 100ms timeout and should be reported as MessageTimedOut. + // Point the producer at an unreachable broker so messages cannot be + // delivered within message.timeout.ms and the delivery callback is + // invoked with MessageTimedOut. Using a real broker is unreliable here + // because Kafka's default auto.create.topics.enable racing with a 100ms + // timeout has produced flaky results on faster CI hardware. let topic_name = rand_test_topic("test_base_producer_timeout"); let producer = base_producer_utils::create_base_producer_with_context( - &kafka_context.bootstrap_servers, + "127.0.0.1:1", context.clone(), &[("message.timeout.ms", "100")], ) From c691df58279fb2d35ec0b587a34968db8599de6a Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Thu, 21 May 2026 17:53:15 -0500 Subject: [PATCH 31/56] test: accept either GroupIdNotFound or NotCoordinator for unknown group Within a test binary, tests share one KafkaContext via OnceCell and default to running in parallel, so the broker's consumer-group coordinator state when test_delete_unknown_group runs depends on whether another test in the binary has touched a group first. A cold broker returns NotCoordinator; a warmed broker returns GroupIdNotFound. Both are correct "the group does not exist" responses, so accept either rather than relying on test ordering. --- tests/consumer_groups.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/consumer_groups.rs b/tests/consumer_groups.rs index 7ac288551..603a688fa 100644 --- a/tests/consumer_groups.rs +++ b/tests/consumer_groups.rs @@ -74,9 +74,25 @@ pub async fn test_delete_unknown_group() { let unknown_group_name = rand_test_group(); let res = admin_client .delete_groups(&[&unknown_group_name], &AdminOptions::default()) - .await; - let expected: GroupResult = Err((unknown_group_name, RDKafkaErrorCode::GroupIdNotFound)); - assert_eq!(res, Ok(vec![expected])); + .await + .expect("delete_groups call failed"); + // The broker reports GroupIdNotFound once the consumer-group coordinator + // has been initialised (any prior test in the binary that touched a + // group is enough), and NotCoordinator on a cold broker. Both indicate + // the same thing for this test: the group does not exist. + let group_result: &GroupResult = res.first().expect("expected one result"); + let (returned_name, code) = group_result + .as_ref() + .expect_err("expected an error for an unknown group"); + assert_eq!(returned_name, &unknown_group_name); + assert!( + matches!( + code, + RDKafkaErrorCode::GroupIdNotFound | RDKafkaErrorCode::NotCoordinator + ), + "unexpected error code: {:?}", + code + ); } /// Verify that deleting a valid and invalid group results in a mixed result From eb3f317b63370cee8c6751fe5be2fe070aa090cd Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 08:19:28 -0500 Subject: [PATCH 32/56] test: wire KAFKA_VERSION into the testcontainers image tag Each CI matrix row passed KAFKA_VERSION (3.7-4.0), but the testcontainers init was using the crate's hard-coded default tag for every run, so the matrix was redundant. Resolve the env var into a specific apache/kafka tag at startup and use the JVM image variant (the kafka-native variant doesn't publish 3.7.x tags). --- tests/utils/containers.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/utils/containers.rs b/tests/utils/containers.rs index 93682f2d6..76e432278 100644 --- a/tests/utils/containers.rs +++ b/tests/utils/containers.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use testcontainers_modules::kafka::apache::Kafka; use testcontainers_modules::testcontainers::core::ContainerPort; use testcontainers_modules::testcontainers::runners::AsyncRunner; -use testcontainers_modules::testcontainers::{ContainerAsync, Image, ImageExt}; +use testcontainers_modules::testcontainers::{ContainerAsync, ImageExt}; use tokio::sync::OnceCell; type KafkaImage = testcontainers_modules::testcontainers::core::ContainerRequest; @@ -52,13 +52,17 @@ impl KafkaContext { } async fn init() -> anyhow::Result> { + let kafka_tag = resolve_kafka_image_tag(); let kafka_container: KafkaImage = Kafka::default() + // The kafka-native image (the crate default) doesn't publish 3.7.x + // tags, so use the JVM image which covers the full CI matrix range. + .with_jvm_image() + .with_tag(&kafka_tag) // The single-broker testcontainers image needs replication and ISR // overrides; otherwise transactions hang because __transaction_state // can't reach its default replication factor of 3. .with_env_var("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1") .with_env_var("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1"); - let kafka_version = Kafka::default().tag().to_string(); let kafka_node = kafka_container .start() @@ -75,10 +79,25 @@ async fn init() -> anyhow::Result> { Ok::, anyhow::Error>(Arc::new(KafkaContext { kafka_node, bootstrap_servers: format!("{}:{}", kafka_host, kafka_port), - version: kafka_version, + version: kafka_tag, })) } +// Map the CI matrix's short KAFKA_VERSION (e.g. "3.7") onto a specific +// apache/kafka tag so each matrix row actually exercises a different broker. +// Without this, the crate's hard-coded default tag would make every row run +// the same image. Full tag strings (e.g. "3.9.1") are passed through. +fn resolve_kafka_image_tag() -> String { + let raw = std::env::var("KAFKA_VERSION").unwrap_or_else(|_| "4.0".into()); + match raw.as_str() { + "3.7" => "3.7.2".into(), + "3.8" => "3.8.1".into(), + "3.9" => "3.9.2".into(), + "4.0" => "4.0.2".into(), + _ => raw, + } +} + #[tokio::test] pub async fn test_kafka_context_works() { let kafka_context_result = KafkaContext::shared().await; From ac9edafaa702723118eebc8365b5112fe9399984 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 09:05:48 -0500 Subject: [PATCH 33/56] docs: add tests/README and prune dead helpers from tests/utils Add a junior-engineer walkthrough of the testcontainers-based suite under tests/README.md, covering: prerequisites, the shared KafkaContext / OnceCell pattern, a worked example of writing a new test, the helper cheatsheet, known quirks (transactions RF override, the unreachable- broker timeout, GroupIdNotFound vs NotCoordinator), the CI matrix, and troubleshooting. Replace the stale "Tests" section in CONTRIBUTING.md with a pointer to it; the old section still described ./test_suite.sh + KAFKA_HOST, which the testcontainers swap obsoleted. Drafting the doc surfaced dead code in tests/utils/mod.rs that was left over from the docker-compose era: - get_bootstrap_server() read the now-unset KAFKA_HOST env var. - populate_topic() had no live callers (all sites were commented out). - create_topic(name, partitions) was shadowed by the new admin::create_topic helper. - ProducerTestContext was only referenced from the dead populate_topic. - consumer_config() had one live caller (test_invalid_max_poll_interval) and was the only thing still wired to get_bootstrap_server. Remove the first four; give consumer_config an explicit bootstrap_servers parameter and update the caller to pass KafkaContext::shared().bootstrap_servers. Also drop two leftover commented-out populate_topic lines in tests/stream_consumers.rs that sat next to their live replacements. No behavior change; cargo fmt / clippy / clippy --tests stay clean. --- CONTRIBUTING.md | 24 +-- tests/README.md | 344 ++++++++++++++++++++++++++++++++++++++ tests/stream_consumers.rs | 9 +- tests/utils/mod.rs | 90 +--------- 4 files changed, 365 insertions(+), 102 deletions(-) create mode 100644 tests/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 987ad1ce2..5c15fa28a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,27 +26,27 @@ The unit tests can run without a Kafka broker present: cargo test --lib ``` -### Automatic testing +### Integration tests -rust-rdkafka contains a suite of tests which is automatically executed by travis in -docker-compose. Given the interaction with C code that rust-rdkafka has to do, tests -are executed in valgrind to check eventual memory errors and leaks. - -To run the full suite using docker-compose: +The integration tests start their own Kafka broker via +[testcontainers-rs], so all you need locally is a running Docker daemon +and the usual Rust toolchain: ```bash -./test_suite.sh +cargo test ``` -To run locally, instead: +To pick a specific Kafka version (default `4.0`), set `KAFKA_VERSION`: ```bash -KAFKA_HOST="kafka_server:9092" cargo test +KAFKA_VERSION=3.9 cargo test ``` -In this case there is a broker expected to be running on `KAFKA_HOST`. -The broker must be configured with default partition number 3 and topic -autocreation in order for the tests to succeed. +For the full walkthrough, including how the shared broker is wired up, +how to add a new test, the helper cheatsheet, and known quirks, see +[`tests/README.md`](tests/README.md). + +[testcontainers-rs]: https://github.com/testcontainers/testcontainers-rs ## Releasing diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..c4d555d91 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,344 @@ +# Integration test suite + +This is a tour of how the integration tests work, aimed at someone new +to the codebase. If you just want to run them, jump to [Running the +tests](#running-the-tests). If you want to add one, read the rest. + +## Overview + +Each test file under `tests/` is a separate Cargo integration-test binary +(`tests/admin.rs`, `tests/base_producer.rs`, and so on). Every binary +needs a real Kafka broker to talk to. + +We start that broker with [testcontainers-rs]. When the first test in a +binary calls `KafkaContext::shared()`, testcontainers pulls and starts +an `apache/kafka` Docker image, waits for it to be ready, and hands back +its bootstrap address (host plus a randomly-allocated port). Every +subsequent call in that same binary reuses the same container. + +You don't need to start the broker yourself. You don't need +`docker-compose`. You don't need to set `KAFKA_HOST`. `cargo test` does +the right thing as long as Docker is running. + +[testcontainers-rs]: https://github.com/testcontainers/testcontainers-rs + +## Prerequisites + +- **A Docker daemon** the current user can talk to (Docker Desktop, + colima, OrbStack, native Docker, ...). Tests fail at + `KafkaContext::shared()` if Docker isn't reachable. +- **Rust >= 1.85.** This is the MSRV the library targets, and the + `testcontainers-modules` dep is pinned at `0.12.1` to keep us on it. +- **librdkafka build deps.** `cmake`, a C/C++ toolchain, plus + `libcurl4-openssl-dev` on Debian/Ubuntu. See the top-level `README.md` + for the full list. + +## Running the tests + +From the repository root: + +```bash +cargo test +``` + +That builds every integration-test binary in `tests/` and runs them +sequentially. The first binary to run pays the cost of pulling the +Kafka image (one-time) and starting a container (a few seconds). Within +a binary, individual tests run in parallel against the shared broker. + +To run just one file: + +```bash +cargo test --test admin +``` + +To run a single test: + +```bash +cargo test --test admin test_topic_create_and_delete +``` + +To pick a specific Kafka version (defaults to `4.0`): + +```bash +KAFKA_VERSION=3.9 cargo test +``` + +`KAFKA_VERSION` accepts either a short series (`3.7`, `3.8`, `3.9`, +`4.0`) or a full tag (`3.9.2`). The mapping lives in +`tests/utils/containers.rs::resolve_kafka_image_tag`. + +To enable librdkafka log output: + +```bash +RUST_LOG="librdkafka=trace,rdkafka::client=debug" cargo test +``` + +## How the broker is managed + +`tests/utils/containers.rs` defines a single `KafkaContext`: + +```rust +pub struct KafkaContext { + kafka_node: ContainerAsync, + pub bootstrap_servers: String, + pub version: String, +} +``` + +`KafkaContext::shared()` is the only constructor. It wraps a +`tokio::sync::OnceCell` so that the first caller in a test binary spins +the container up and every other caller gets the same `Arc` +back: + +```rust +let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); +``` + +A few things to internalise: + +- **One broker per test binary.** Cargo runs each `tests/*.rs` file as + a separate process, so each binary gets its own container. Two tests + in the same binary share state on the broker. Two tests in different + binaries do not. +- **Tests in a binary run in parallel.** That's Cargo's default. If your + test needs an isolated topic or consumer group, use the random-name + helpers (see below). Don't hardcode names; you will collide with + another test. +- **No fresh broker between tests.** The container lives for the whole + binary. State you leave behind (topics, consumer groups, offsets) is + visible to later tests in the same binary, which is occasionally what + you want and occasionally a footgun. +- **Random host port.** `kafka_context.bootstrap_servers` is something + like `127.0.0.1:54731`. Never assume `9092`. + +The container is started with two non-default env vars: + +```rust +.with_env_var("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1") +.with_env_var("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1") +``` + +Kafka defaults to replication factor 3 for `__transaction_state`. We +only have a single broker, so the transactions tests would hang forever +waiting for the internal topic to come up without these overrides. If +you ever see transactions tests hang, this is the first thing to check. + +We also call `.with_jvm_image()` because the kafka-native variant of the +`apache/kafka` image doesn't publish `3.7.x` tags, and the CI matrix +needs them. + +## Test file layout + +``` +tests/ + admin.rs integration tests, one file per area + base_consumer.rs + base_producer.rs + consumer_groups.rs + future_producer.rs + metadata.rs + producer.rs + stream_consumers.rs + topic_partition_lists.rs (pure unit-style; no broker) + transactions.rs + + utils/ shared test helpers + mod.rs message-production helpers, + ConsumerTestContext, KafkaVersion, + BROKER_ID (= 1, the container's hardcoded id) + containers.rs KafkaContext + shared() / OnceCell + admin.rs create_admin_client, create_topic, new_topic_vec + consumer/ + mod.rs base-consumer helpers + stream_consumer.rs stream-consumer helpers + producer/ + mod.rs + base_producer.rs create_producer, send_record, poll_and_flush + future_producer.rs create_producer (FutureProducer) + topics.rs populate_topic_using_future_producer + rand.rs rand_test_topic / rand_test_group / + rand_test_transactional_id + logging.rs init_test_logger (env_logger, one-shot) +``` + +Each test file lives at the top of `tests/`. Helpers live under +`tests/utils/` and are pulled in with `mod utils;` at the top of each +file. The split lets Cargo treat each integration test as its own +binary while sharing utility code. + +## Writing a new integration test + +Start with a small example. Drop a new file at `tests/my_feature.rs`: + +```rust +use rdkafka::admin::AdminOptions; + +use crate::utils::admin; +use crate::utils::containers::KafkaContext; +use crate::utils::logging::init_test_logger; +use crate::utils::producer; +use crate::utils::rand::{rand_test_group, rand_test_topic}; + +mod utils; + +#[tokio::test] +async fn my_feature_works() { + init_test_logger(); + + // 1. Get (or start, if we're the first test) the shared broker. + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + // 2. Create an admin client and a unique topic for this test. + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + + let topic_name = rand_test_topic("my_feature"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + // 3. Do the thing under test. Make assertions. + let producer = producer::future_producer::create_producer( + &kafka_context.bootstrap_servers, + ) + .await + .expect("could not create producer"); + + // ... produce, consume, assert ... + + drop(producer); +} +``` + +Things to notice: + +- `mod utils;` at the top is mandatory; without it the `crate::utils::*` + paths don't resolve. +- `init_test_logger()` is a one-shot guarded by `Once`, so it's safe to + call from every test. +- `rand_test_topic("my_feature")` returns `my_feature_aB3xQ...`, a + test-specific topic. Always do this for topics and consumer groups, + or you'll see flaky tests when binaries are run in parallel locally. +- Pass `&kafka_context.bootstrap_servers` into every helper that builds + a client. There is no global "bootstrap servers" anywhere; the + container's address is only known after `shared()` resolves. + +## Helper cheatsheet + +Most of what you'll need is in `tests/utils/`: + +| You want to ... | Use | +| -------------------------------- | ------------------------------------------------------------------ | +| Get the shared broker | `containers::KafkaContext::shared().await` | +| Make an admin client | `admin::create_admin_client(bootstrap_servers).await` | +| Create a topic | `admin::create_topic(client, name).await` (one partition, RF 1) | +| Get a `NewTopic` vec for finer control | `admin::new_topic_vec(name, Some(num_partitions))` | +| Make a `BaseProducer` | `producer::base_producer::create_producer(bootstrap_servers).await` | +| Make a `FutureProducer` | `producer::future_producer::create_producer(bootstrap_servers).await` | +| Make a `BaseConsumer` | `consumer::create_subscribed_base_consumer(bootstrap_servers, group, topic).await` | +| Make a `StreamConsumer` | `consumer::stream_consumer::create_stream_consumer(bootstrap_servers, Some(group)).await` | +| Produce N messages | `produce_messages(producer, topic, n, partition, timestamp).await` (from `utils::*`) | +| Produce N to a partition | `produce_messages_to_partition(producer, topic, n, partition).await` | +| Populate a topic with a `FutureProducer` | `topics::populate_topic_using_future_producer(producer, topic, n, partition).await` | +| Random topic name | `rand::rand_test_topic("test_name")` | +| Random consumer group | `rand::rand_test_group()` | +| Random transactional id | `rand::rand_test_transactional_id()` | +| Init `env_logger` once | `logging::init_test_logger()` | + +Constants you'll see: + +- `utils::BROKER_ID == 1`. The single-broker testcontainers image + hardcodes its broker id to 1. Assert against the constant, not a + magic number. + +## Known quirks + +A few tests have non-obvious shapes; read these before debugging a +failure you didn't introduce. + +- **`tests/base_producer.rs::test_base_producer_timeout`** points the + producer at `127.0.0.1:1` (a deliberately unreachable address) instead + of the real broker. The test is exercising the delivery callback when + `message.timeout.ms` fires. Pointing at the real broker, with + `auto.create.topics.enable=true`, lets the message get delivered + inside the 100ms deadline on fast CI hardware and the test flakes. + +- **`tests/consumer_groups.rs::test_delete_unknown_group`** accepts + either `GroupIdNotFound` or `NotCoordinator`. Both mean "the group + doesn't exist," but which one you get depends on whether any earlier + test in the binary has caused the consumer-group coordinator to + initialise. We share a broker across the file, so coordinator state + is non-deterministic from this test's point of view. + +- **`tests/metadata.rs`** uses `BROKER_ID = 1` (the testcontainers + default) and does not assert on host port. The old test suite + hardcoded `0` and `9092` for the docker-compose broker; both are wrong + here. + +- **Transactions tests need the replication overrides.** If you ever + copy `containers.rs::init` and drop the + `KAFKA_TRANSACTION_STATE_LOG_*` env vars, the transactions tests will + hang on broker startup waiting for `__transaction_state` to reach RF + 3. They won't fail with a clear error; they'll just sit there until + the timeout. + +## CI + +`.github/workflows/ci.yml` runs four jobs: + +- **lint**: `cargo fmt --check`, `cargo clippy -- -Dwarnings`, + `cargo clippy --tests -- -Dwarnings`, `cargo test --doc`. Lint + failures break the build. Always run these locally before pushing. +- **check**: cross-platform builds on macOS, Windows, and Ubuntu with + various feature combinations. No tests, just `cargo build` and + `cargo test` in `rdkafka-sys` (its tests don't need a broker). +- **check-minimal-versions**: makes sure the declared semver + constraints actually resolve. +- **test**: the integration suite. Fans out across `KAFKA_VERSION = + 3.7, 3.8, 3.9, 4.0`. Each row resolves to a specific + `apache/kafka:` via `resolve_kafka_image_tag`. Rows run + sequentially (`max-parallel: 1`) because they share an Actions runner + and each spawns its own Docker container. + +The test job currently runs through `test_suite.sh`, which still has a +leftover `docker compose up --wait` from the previous setup. The +docker-compose broker is no longer used by any test; removing it from +the script and inlining the rest of the script into `ci.yml` is a +straightforward follow-up. + +## Troubleshooting + +**"could not create kafka context" / Docker errors.** The Docker +daemon isn't running, or the current user can't reach it. Start Docker +Desktop / colima / OrbStack and retry. + +**Image pull is slow on first run.** Expected. testcontainers pulls +`apache/kafka:` on first use and caches it in your local Docker +image store. Subsequent runs reuse it. + +**Test hangs on the transactions suite.** Almost certainly the +`__transaction_state` topic can't reach its replication factor. Check +that the `KAFKA_TRANSACTION_STATE_LOG_*` env vars are still set in +`containers.rs::init`. + +**Port 9092 in use.** Doesn't matter to testcontainers (it allocates a +random host port), but a stale docker-compose broker from before this +suite can confuse things if you also run examples that hardcode +`localhost:9092`. `docker compose down` clears it. + +**A test passes locally but fails in CI on Kafka 3.7.** The broker is a +different version. Run the same matrix locally: `KAFKA_VERSION=3.7 +cargo test --test `. + +**Clippy complains in CI but not locally.** The lint job runs +`cargo clippy --tests -- -Dwarnings`. Reproduce that flag set locally. diff --git a/tests/stream_consumers.rs b/tests/stream_consumers.rs index 8c40723f3..dab35d16b 100644 --- a/tests/stream_consumers.rs +++ b/tests/stream_consumers.rs @@ -28,7 +28,14 @@ mod utils; #[tokio::test] async fn test_invalid_max_poll_interval() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let res: Result = consumer_config( + &kafka_context.bootstrap_servers, &crate::utils::rand::rand_test_group(), Some(hashmap! { "max.poll.interval.ms" => "-1" }), ) @@ -75,7 +82,6 @@ async fn test_produce_consume_base() { ) .await .expect("Could not populate topic using Future producer"); - // let message_map = populate_topic(&topic_name, 100, &value_fn, &key_fn, None, None).await; let consumer = utils::consumer::stream_consumer::create_stream_consumer( &kafka_context.bootstrap_servers, Some(&rand_test_group()), @@ -135,7 +141,6 @@ async fn test_produce_consume_base_concurrent() { ) .await .expect("Could not populate topic using Future producer"); - // let message_map = populate_topic(&topic_name, 100, &value_fn, &key_fn, None, None).await; let consumer = Arc::new( consumer::stream_consumer::create_stream_consumer( &kafka_context.bootstrap_servers, diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index e68caa5cf..3b466afda 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -9,28 +9,20 @@ pub mod rand; pub mod topics; use std::collections::HashMap; -use std::env::{self}; -use std::time::Duration; use regex::Regex; use crate::utils::containers::KafkaContext; -use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication}; use rdkafka::client::ClientContext; use rdkafka::config::ClientConfig; use rdkafka::consumer::ConsumerContext; use rdkafka::error::KafkaResult; -use rdkafka::message::ToBytes; use rdkafka::producer::{FutureProducer, FutureRecord}; use rdkafka::statistics::Statistics; use rdkafka::TopicPartitionList; pub const BROKER_ID: i32 = 1; -pub fn get_bootstrap_server() -> String { - env::var("KAFKA_HOST").unwrap_or_else(|_| "localhost:9092".to_owned()) -} - pub fn get_broker_version(kafka_context: &KafkaContext) -> KafkaVersion { let regex = Regex::new(r"^(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?$").unwrap(); match regex.captures(&kafka_context.version) { @@ -50,85 +42,6 @@ pub fn get_broker_version(kafka_context: &KafkaContext) -> KafkaVersion { #[derive(Debug, Eq, PartialEq, Ord, PartialOrd)] pub struct KafkaVersion(pub u32, pub u32, pub u32, pub u32); -pub struct ProducerTestContext { - _some_data: i64, // Add some data so that valgrind can check proper allocation -} - -impl ClientContext for ProducerTestContext { - fn stats(&self, _: Statistics) {} // Don't print stats -} - -pub async fn create_topic(name: &str, partitions: i32) { - let client: AdminClient<_> = consumer_config("create_topic", None).create().unwrap(); - client - .create_topics( - &[NewTopic::new(name, partitions, TopicReplication::Fixed(1))], - &AdminOptions::new(), - ) - .await - .unwrap(); -} - -/// Produce the specified count of messages to the topic and partition specified. A map -/// of (partition, offset) -> message id will be returned. It panics if any error is encountered -/// while populating the topic. -pub async fn populate_topic( - topic_name: &str, - count: i32, - value_fn: &P, - key_fn: &K, - partition: Option, - timestamp: Option, -) -> HashMap<(i32, i64), i32> -where - P: Fn(i32) -> J, - K: Fn(i32) -> Q, - J: ToBytes, - Q: ToBytes, -{ - let prod_context = ProducerTestContext { _some_data: 1234 }; - - // Produce some messages - let producer = &ClientConfig::new() - .set("bootstrap.servers", get_bootstrap_server().as_str()) - .set("statistics.interval.ms", "500") - .set("debug", "all") - .set("message.timeout.ms", "30000") - .create_with_context::>(prod_context) - .expect("Producer creation error"); - - let futures = (0..count) - .map(|id| { - let future = async move { - producer - .send( - FutureRecord { - topic: topic_name, - payload: Some(&value_fn(id)), - key: Some(&key_fn(id)), - partition, - timestamp, - headers: None, - }, - Duration::from_secs(1), - ) - .await - }; - (id, future) - }) - .collect::>(); - - let mut message_map = HashMap::new(); - for (id, future) in futures { - match future.await { - Ok(delivered) => message_map.insert((delivered.partition, delivered.offset), id), - Err((kafka_error, _message)) => panic!("Delivery failed: {}", kafka_error), - }; - } - - message_map -} - pub async fn produce_messages_with_timestamp( producer: &FutureProducer, topic_name: &str, @@ -225,6 +138,7 @@ impl ConsumerContext for ConsumerTestContext { } pub fn consumer_config( + bootstrap_servers: &str, group_id: &str, config_overrides: Option>, ) -> ClientConfig { @@ -232,7 +146,7 @@ pub fn consumer_config( config.set("group.id", group_id); config.set("client.id", "rdkafka_integration_test_client"); - config.set("bootstrap.servers", get_bootstrap_server().as_str()); + config.set("bootstrap.servers", bootstrap_servers); config.set("enable.partition.eof", "false"); config.set("session.timeout.ms", "6000"); config.set("enable.auto.commit", "false"); From 8a8c5cadb4a41f7bae0b0202725708aacbd76f01 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 09:07:19 -0500 Subject: [PATCH 34/56] ci: drop test_suite.sh + docker-compose, call cargo test directly The integration tests own their broker through testcontainers now, so test_suite.sh's docker-compose orchestration is no longer load-bearing: the Bitnami broker it spun up was never reached by any test in the new suite. Inline the only step that matters (cargo test, with KAFKA_VERSION passed through) into the CI test job and delete the surrounding files: - delete docker-compose.yaml (Bitnami Kafka container; unused). - delete test_suite.sh (its only live work was `cargo test` plus the smol / async-std example smoke runs; see below). - the explicit `git submodule update --init` it ran is redundant with rdkafka-sys's build.rs, which initialises the librdkafka submodule itself. The other CI jobs already rely on this. One coverage item drops with the script: the smol and async-std runtime examples used to be invoked at the end of test_suite.sh as e2e smoke tests against the docker-compose broker. The `check` job still builds them via `cargo build --all-targets`, so compile coverage is preserved; runtime coverage is not. If we want it back it belongs in its own job (a service-container broker plus two `cargo run --example ...` steps) rather than entangled with the integration suite. Also drop the matching "easy follow-up" note from tests/README.md. --- .github/workflows/ci.yml | 9 ++--- docker-compose.yaml | 23 ------------ test_suite.sh | 77 ---------------------------------------- tests/README.md | 19 ++++------ 4 files changed, 12 insertions(+), 116 deletions(-) delete mode 100644 docker-compose.yaml delete mode 100755 test_suite.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d99a20fa..e02e1e663 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,8 @@ jobs: test: strategy: fail-fast: false - # The test suite doesn't support concurrent runs. + # Each row spins up its own Docker container via testcontainers, so + # keep them sequential to avoid contention on a single runner. max-parallel: 1 matrix: include: @@ -88,8 +89,8 @@ jobs: toolchain: ${{ env.rust_version }} - run: sudo apt-get update - run: sudo apt-get install -y libcurl4-openssl-dev - # - run: sudo apt-get install -qy valgrind # Valgrind currently disabled in testing - - run: ./test_suite.sh + - run: cargo test env: KAFKA_VERSION: ${{ matrix.kafka-version }} - TERM: xterm-256color + RUST_LOG: off + RUST_BACKTRACE: 1 diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index dcbd43155..000000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,23 +0,0 @@ -services: - kafka: - image: bitnamilegacy/kafka:${KAFKA_VERSION:-4.0} - environment: - # Enable KRaft mode (combined broker and controller) - - KAFKA_CFG_NODE_ID=0 - - KAFKA_CFG_BROKER_ID=0 # In KRaft, this should be the same as the node ID - - KAFKA_CFG_PROCESS_ROLES=broker,controller - - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER - - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 - - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 - - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT - - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@localhost:9093 - - # Bitnami defaults - - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 - - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=1 - - KAFKA_CFG_NUM_PARTITIONS=3 - - # This is a Bitnami-specific variable to disable ZooKeeper - - KAFKA_KRAFT_ENABLED=true - ports: ["9092:9092"] diff --git a/test_suite.sh b/test_suite.sh deleted file mode 100755 index 0ff14c926..000000000 --- a/test_suite.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -set -euo pipefail - -echo_good() { - tput setaf 2 - echo "$@" - tput sgr0 -} - -echo_bad() { - tput setaf 1 - echo "$@" - tput sgr0 -} - -run_with_valgrind() { - if ! valgrind --error-exitcode=100 --suppressions=rdkafka.suppressions --gen-suppressions=all --leak-check=full "$1" --nocapture --test-threads=1 - then - echo_bad "*** Failure in $1 ***" - exit 1 - fi -} - -# Initialize. - -git submodule update --init -docker compose up --wait - -# Run integration tests -export RUST_LOG=${RUST_LOG:-off} -RUST_BACKTRACE=1 cargo test "$@" - - -# Run unit tests. - -#echo_good "*** Run unit tests ***" -#for test_file in target/debug/deps/rdkafka-* -#do -# if [[ -x "$test_file" ]] -# then -# echo_good "Executing "$test_file"" -# run_with_valgrind "$test_file" -# fi -#done -#echo_good "*** Unit tests succeeded ***" -# -## Run integration tests. -# -#echo_good "*** Run integration tests ***" -#for test_file in target/debug/deps/test_* -#do -# if [[ -x "$test_file" ]] -# then -# #echo_good "*** Restarting kafka/zk ***" -# #docker-compose restart --timeout 30 -# echo_good "Executing "$test_file"" -# run_with_valgrind "$test_file" -# fi -#done -#echo_good "*** Integration tests succeeded ***" - -# Run smol runtime example. - -echo_good "*** Run runtime_smol example ***" -cargo run --example runtime_smol --no-default-features --features cmake-build -- --topic smol -echo_good "*** runtime_smol example succeeded ***" - -# Run async-std runtime example. - -echo_good "*** Run runtime_async_std example ***" -cargo run --example runtime_async_std --no-default-features --features cmake-build -- --topic async-std -echo_good "*** runtime_async_std example succeeded ***" diff --git a/tests/README.md b/tests/README.md index c4d555d91..cb715d652 100644 --- a/tests/README.md +++ b/tests/README.md @@ -306,15 +306,9 @@ failure you didn't introduce. constraints actually resolve. - **test**: the integration suite. Fans out across `KAFKA_VERSION = 3.7, 3.8, 3.9, 4.0`. Each row resolves to a specific - `apache/kafka:` via `resolve_kafka_image_tag`. Rows run - sequentially (`max-parallel: 1`) because they share an Actions runner - and each spawns its own Docker container. - -The test job currently runs through `test_suite.sh`, which still has a -leftover `docker compose up --wait` from the previous setup. The -docker-compose broker is no longer used by any test; removing it from -the script and inlining the rest of the script into `ci.yml` is a -straightforward follow-up. + `apache/kafka:` via `resolve_kafka_image_tag` and runs + `cargo test`. Rows run sequentially (`max-parallel: 1`) because they + share an Actions runner and each spawns its own Docker container. ## Troubleshooting @@ -332,9 +326,10 @@ that the `KAFKA_TRANSACTION_STATE_LOG_*` env vars are still set in `containers.rs::init`. **Port 9092 in use.** Doesn't matter to testcontainers (it allocates a -random host port), but a stale docker-compose broker from before this -suite can confuse things if you also run examples that hardcode -`localhost:9092`. `docker compose down` clears it. +random host port), but examples under `examples/` default to +`localhost:9092`. If you're running an example against a separate +broker you started by hand, make sure nothing else is bound to that +port. **A test passes locally but fails in CI on Kafka 3.7.** The broker is a different version. Run the same matrix locally: `KAFKA_VERSION=3.7 From 222a59a9f4bf201dbf67310c834bd2b1fee85928 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 09:12:01 -0500 Subject: [PATCH 35/56] chore: remove dead Dockerfile and valgrind suppressions Two dangling pieces of the pre-testcontainers test infrastructure that nothing in the tree now references: - Dockerfile: copied a docker/run_tests.sh that doesn't exist, set KAFKA_HOST=kafka:9092 (a dead env var after the testcontainers swap), and pinned Rust 1.74 (the MSRV is 1.85). No build, workflow, or README in the repo invokes it. - rdkafka.suppressions: valgrind suppression file. Its only caller was the run_with_valgrind helper inside the just-deleted test_suite.sh, and that helper was itself only reachable from an already-commented- out block. --- Dockerfile | 30 ------------------------------ rdkafka.suppressions | 22 ---------------------- 2 files changed, 52 deletions(-) delete mode 100644 Dockerfile delete mode 100644 rdkafka.suppressions diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d7047b995..000000000 --- a/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM ubuntu:24.10 - -RUN apt-get update && apt-get install -y build-essential \ - curl \ - openssl libssl-dev \ - pkg-config \ - python \ - valgrind \ - zlib1g-dev - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.74 -ENV PATH=/root/.cargo/bin/:$PATH - -# # Create dummy project for rdkafka -# COPY Cargo.toml /rdkafka/ -# RUN mkdir -p /rdkafka/src && echo "fn main() {}" > /rdkafka/src/main.rs -# -# # Create dummy project for rdkafka -# RUN mkdir /rdkafka/rdkafka-sys -# COPY rdkafka-sys/Cargo.toml /rdkafka/rdkafka-sys -# RUN mkdir -p /rdkafka/rdkafka-sys/src && touch /rdkafka/rdkafka-sys/src/lib.rs -# RUN echo "fn main() {}" > /rdkafka/rdkafka-sys/build.rs -# -# RUN cd /rdkafka && test --no-run - -COPY docker/run_tests.sh /rdkafka/ - -ENV KAFKA_HOST=kafka:9092 - -WORKDIR /rdkafka diff --git a/rdkafka.suppressions b/rdkafka.suppressions deleted file mode 100644 index ed248e58b..000000000 --- a/rdkafka.suppressions +++ /dev/null @@ -1,22 +0,0 @@ -# Valgrind suppression file. - -# Spurious statx complaints: https://github.com/rust-lang/rust/issues/68979. -# TODO(benesch): remove when CI upgrades to Valgrind 3.16. -{ - - Memcheck:Param - statx(file_name) - fun:statx - fun:statx - fun:_ZN3std3sys4unix2fs9try_statx* - ... -} -{ - - Memcheck:Param - statx(buf) - fun:statx - fun:statx - fun:_ZN3std3sys4unix2fs9try_statx* - ... -} From 0866fbcd1df72dfad21196e8e13b18d686962567 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 09:24:02 -0500 Subject: [PATCH 36/56] chore: remove broken coverage.sh kcov wrapper that hasn't kept up with the codebase. It globs target/debug/test_* for integration test binaries, which matched the pre-PR-798 layout (test_admin.rs, test_consumer_*) but matches nothing now that integration tests live in per-area files (admin.rs, base_producer.rs, etc.). Nothing in CI or the repo's docs invokes it; the breakage went unnoticed, which is a fair signal that no one is actually running it. Modern alternative is `cargo llvm-cov` if someone wants coverage locally; easier to reach for than maintaining a bespoke kcov script. --- coverage.sh | 69 ----------------------------------------------------- 1 file changed, 69 deletions(-) delete mode 100755 coverage.sh diff --git a/coverage.sh b/coverage.sh deleted file mode 100755 index 83ec20a81..000000000 --- a/coverage.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -GREEN='\033[0;32m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -INCLUDE="/src" -EXCLUDE="/.cargo,rdkafka-sys/librdkafka,rdkafka-sys/src/bindings" -TARGET="target/cov" - -KCOV_ARGS="--include-pattern=$INCLUDE --exclude-pattern=$EXCLUDE --verify $TARGET" - -RDKAFKA_UNIT_TESTS="target/debug/rdkafka-" -RDKAFKASYS_UNIT_TESTS="rdkafka-sys/target/debug/rdkafka_sys-" -INTEGRATION_TESTS="target/debug/test_" - -export RUSTFLAGS="-C link-dead-code" - -echo -e "${GREEN}*** Clean previous coverage results and executables ***${NC}" -rm -rf "$TARGET" -rm -f "$RDKAFKA_UNIT_TESTS"* -rm -f "$RDKAFKASYS_UNIT_TESTS"* -rm -f "$INTEGRATION_TESTS"* - -echo -e "${GREEN}*** Rebuilding tests ***${NC}" -cargo test --no-run -pushd rdkafka-sys && cargo test --no-run && popd - -echo -e "${GREEN}*** Run coverage on rdkafka unit tests ***${NC}" -for test_file in `ls "$RDKAFKA_UNIT_TESTS"*` -do - if [[ ! -x "$test_file" ]]; then - continue - fi - kcov $KCOV_ARGS "$test_file" - if [ "$?" != "0" ]; then - echo -e "${RED}*** Failure during unit test converage ***${NC}" - exit 1 - fi -done - -echo -e "${GREEN}*** Run coverage on rdkafka-sys unit tests ***${NC}" -for test_file in `ls "$RDKAFKASYS_UNIT_TESTS"*` -do - if [[ ! -x "$test_file" ]]; then - continue - fi - kcov $KCOV_ARGS "$test_file" - if [ "$?" != "0" ]; then - echo -e "${RED}*** Failure during rdkafka-sys unit test converage ***${NC}" - exit 1 - fi -done - -echo -e "${GREEN}*** Run coverage on rdkafka integration tests ***${NC}" -for test_file in `ls "$INTEGRATION_TESTS"*` -do - if [[ ! -x "$test_file" ]]; then - continue - fi - echo -e "${GREEN}Executing "$test_file"${NC}" - kcov $KCOV_ARGS "$test_file" - if [ "$?" != "0" ]; then - echo -e "${RED}*** Failure during integration converage ***${NC}" - exit 1 - fi -done - -echo -e "${GREEN}*** Coverage completed successfully ***${NC}" From 7021be032bb749cae9154bdd658adfe35ebfecbc Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 09:30:43 -0500 Subject: [PATCH 37/56] ci: add runtime-examples job for the smol and async-std examples Bring back the e2e smoke test for the alternative runtime examples that test_suite.sh used to run. Putting it in its own job, with a service-container broker, instead of re-entangling it with the integration suite: - the integration test job uses testcontainers and a random host port, so it can't host examples that hardcode localhost:9092; - the examples are runtime-correctness checks for smol and async-std, not broker-compatibility checks, so a single pinned Kafka version is enough and matrixing across 3.7-4.0 would be wasted CI time. The service container is apache/kafka:4.0.2 in KRaft mode with the same single-broker overrides we set in tests/utils/containers.rs (transaction-state-log RF/ISR = 1), exposed on localhost:9092 with a kafka-topics.sh-based healthcheck so the runner waits for the broker before launching cargo. The examples produce / consume one message each and exit non-zero on failure. Verified locally that both examples compile with `--no-default-features --features cmake-build`. The check job's cargo build --all-targets continues to catch pure compile breakage; this job catches anything that only manifests at runtime. --- .github/workflows/ci.yml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e02e1e663..138a63b03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,3 +94,42 @@ jobs: KAFKA_VERSION: ${{ matrix.kafka-version }} RUST_LOG: off RUST_BACKTRACE: 1 + + # Smoke-test the smol and async-std runtime examples against a real + # broker. The integration suite covers the tokio path via testcontainers; + # this job catches breakage in the alternative runtimes that + # `cargo build --all-targets` would miss. + runtime-examples: + runs-on: ubuntu-24.04 + services: + kafka: + image: apache/kafka:4.0.2 + ports: + - 9092:9092 + env: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + options: >- + --health-cmd "/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + --health-start-period 30s + steps: + - uses: actions/checkout@v4 + - uses: lukka/get-cmake@latest + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ env.rust_version }} + - run: sudo apt-get update + - run: sudo apt-get install -y libcurl4-openssl-dev + - run: cargo run --example runtime_smol --no-default-features --features cmake-build -- --topic smol + - run: cargo run --example runtime_async_std --no-default-features --features cmake-build -- --topic async-std From 3ade2c503f0a2cd310c2c3649849bec02f350363 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 09:53:46 -0500 Subject: [PATCH 38/56] test: tolerate the apache/kafka 3.7 startup wakeup on the split queue test_produce_consume_message_queue_nonempty_callback asserts wakeup counts on a split_partition_queue, with the partition assigned at Offset::Beginning. On apache/kafka 3.7.x (JVM image), librdkafka's initial log-start-offset query posts an event to that queue during setup, which invokes the nonempty callback once before any messages have been produced. 3.8 / 3.9 / 4.0 don't show this. The old Bitnami 3.7 image apparently didn't either, which is why the test passed on the pre-matrix-fix CI matrix that was always pulling the 4.x default. Capture the wakeup count after initial setup as a baseline and assert deltas (baseline + N) instead of absolute counts. The test still verifies the callback's actual contract -- one wakeup per empty-to- non-empty transition -- without depending on broker-version-specific startup events. --- tests/base_consumer.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/base_consumer.rs b/tests/base_consumer.rs index 3b0e4f61c..4035f4a19 100644 --- a/tests/base_consumer.rs +++ b/tests/base_consumer.rs @@ -521,9 +521,15 @@ async fn test_produce_consume_message_queue_nonempty_callback() { // Initiate connection. assert!(consumer.poll(Duration::from_secs(0)).is_none()); - // Expect no wakeups for 1s. + // Let any startup events drain through. apache/kafka 3.7.x posts an + // event to the split partition queue during initial position setup + // (the partition is assigned at Offset::Beginning, so librdkafka has + // to query the log start offset), which invokes the nonempty + // callback once before any messages exist. 3.8+ doesn't show this. + // Capture the post-setup wakeup count as our baseline and assert + // deltas from here on. thread::sleep(Duration::from_secs(1)); - assert_eq!(wakeups.load(Ordering::SeqCst), 0); + let baseline = wakeups.load(Ordering::SeqCst); // Verify there are no messages waiting. assert!(consumer.poll(Duration::from_secs(0)).is_none()); @@ -534,7 +540,7 @@ async fn test_produce_consume_message_queue_nonempty_callback() { .await .expect("Could not create Future producer"); produce_messages(&producer, &topic_name, 2, None, None).await; - wait_for_wakeups(1); + wait_for_wakeups(baseline + 1); // Read one of the messages. assert!(queue.poll(Duration::from_secs(0)).is_some()); @@ -543,7 +549,7 @@ async fn test_produce_consume_message_queue_nonempty_callback() { // queue is not fully drained, for 1s. produce_messages(&producer, &topic_name, 2, None, None).await; thread::sleep(Duration::from_secs(1)); - assert_eq!(wakeups.load(Ordering::SeqCst), 1); + assert_eq!(wakeups.load(Ordering::SeqCst), baseline + 1); // Drain the queue. assert!(queue.poll(None).is_some()); @@ -552,15 +558,15 @@ async fn test_produce_consume_message_queue_nonempty_callback() { // Expect no additional wakeups for 1s. thread::sleep(Duration::from_secs(1)); - assert_eq!(wakeups.load(Ordering::SeqCst), 1); + assert_eq!(wakeups.load(Ordering::SeqCst), baseline + 1); // Add another message, and expect a wakeup. produce_messages(&producer, &topic_name, 1, None, None).await; - wait_for_wakeups(2); + wait_for_wakeups(baseline + 2); // Expect no additional wakeups for 1s. thread::sleep(Duration::from_secs(1)); - assert_eq!(wakeups.load(Ordering::SeqCst), 2); + assert_eq!(wakeups.load(Ordering::SeqCst), baseline + 2); // Disable the queue and add another message. queue.set_nonempty_callback(|| ()); @@ -568,7 +574,7 @@ async fn test_produce_consume_message_queue_nonempty_callback() { // Expect no additional wakeups for 1s. thread::sleep(Duration::from_secs(1)); - assert_eq!(wakeups.load(Ordering::SeqCst), 2); + assert_eq!(wakeups.load(Ordering::SeqCst), baseline + 2); } //TODO: adjust the test to work, today set_nonempty_callback param is never called. From aab163aa0bfc478e73560cfe65918158372541ca Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 09:53:46 -0500 Subject: [PATCH 39/56] docs: tests/README: add runtime-examples job and 3.7 wakeup quirk Two doc-sync items missed in the preceding commits: - The CI section listed four jobs; add the new runtime-examples job introduced for the smol / async-std example smoke tests. - The Known quirks section gains an entry for the apache/kafka 3.7 startup wakeup on the split partition queue, mirroring the test comment in test_produce_consume_message_queue_nonempty_callback. --- tests/README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index cb715d652..1ef86ff50 100644 --- a/tests/README.md +++ b/tests/README.md @@ -292,9 +292,17 @@ failure you didn't introduce. 3. They won't fail with a clear error; they'll just sit there until the timeout. +- **`tests/base_consumer.rs::test_produce_consume_message_queue_nonempty_callback`** + asserts wakeup-count deltas against a baseline captured after initial + setup, not absolute counts. apache/kafka 3.7.x posts an event to the + split partition queue during the initial position query (the partition + is assigned at `Offset::Beginning`), which fires the nonempty callback + once before any messages are produced. 3.8+ doesn't. Comparing deltas + is portable across the matrix. + ## CI -`.github/workflows/ci.yml` runs four jobs: +`.github/workflows/ci.yml` runs five jobs: - **lint**: `cargo fmt --check`, `cargo clippy -- -Dwarnings`, `cargo clippy --tests -- -Dwarnings`, `cargo test --doc`. Lint @@ -309,6 +317,12 @@ failure you didn't introduce. `apache/kafka:` via `resolve_kafka_image_tag` and runs `cargo test`. Rows run sequentially (`max-parallel: 1`) because they share an Actions runner and each spawns its own Docker container. +- **runtime-examples**: smoke-tests `examples/runtime_smol.rs` and + `examples/runtime_async_std.rs` against a pinned `apache/kafka:4.0.2` + service container. The integration suite covers the tokio path via + testcontainers; this job catches breakage in the alternative runtimes + that `cargo build --all-targets` would miss. Not matrixed: it's a + runtime-correctness check, not a broker-compatibility check. ## Troubleshooting From 31e17b2aee739fa2e4e6e29c495e378bd3cf9568 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:08:24 -0500 Subject: [PATCH 40/56] test: cover compression round-trip for gzip, snappy, lz4, and zstd Adds four FutureProducer round-trip tests in tests/future_producer.rs, one per librdkafka-supported compression codec. Each test produces 64 messages with a non-trivial payload through a producer configured with `compression.type=`, then consumes them back through a stream consumer and asserts every key/value pair survives the codec. The zstd test is gated on `#[cfg(feature = "zstd")]` because rdkafka-sys's build.rs configures librdkafka with `--disable-zstd` unless the `zstd` Cargo feature is enabled. The CI integration-test job is updated to pass `--features zstd` so the matrix actually exercises that codec. Adds a `create_producer_with_overrides` helper to tests/utils/producer/future_producer.rs and documents it (plus the new `--features zstd` test invocation) in tests/README.md. --- .github/workflows/ci.yml | 7 +- tests/README.md | 9 +- tests/future_producer.rs | 110 ++++++++++++++++++++++++ tests/utils/producer/future_producer.rs | 10 +++ 4 files changed, 133 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 138a63b03..bb4bbc8fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,12 @@ jobs: toolchain: ${{ env.rust_version }} - run: sudo apt-get update - run: sudo apt-get install -y libcurl4-openssl-dev - - run: cargo test + # `--features zstd` so the zstd compression round-trip test in + # `tests/future_producer.rs` actually links zstd. The default build + # passes `--disable-zstd` to librdkafka (see rdkafka-sys/build.rs), + # which would make a `compression.type=zstd` producer reject the + # config at creation. + - run: cargo test --features zstd env: KAFKA_VERSION: ${{ matrix.kafka-version }} RUST_LOG: off diff --git a/tests/README.md b/tests/README.md index 1ef86ff50..c75256d19 100644 --- a/tests/README.md +++ b/tests/README.md @@ -245,6 +245,7 @@ Most of what you'll need is in `tests/utils/`: | Get a `NewTopic` vec for finer control | `admin::new_topic_vec(name, Some(num_partitions))` | | Make a `BaseProducer` | `producer::base_producer::create_producer(bootstrap_servers).await` | | Make a `FutureProducer` | `producer::future_producer::create_producer(bootstrap_servers).await` | +| `FutureProducer` with config overrides | `producer::future_producer::create_producer_with_overrides(bootstrap_servers, &[(key, value)]).await` | | Make a `BaseConsumer` | `consumer::create_subscribed_base_consumer(bootstrap_servers, group, topic).await` | | Make a `StreamConsumer` | `consumer::stream_consumer::create_stream_consumer(bootstrap_servers, Some(group)).await` | | Produce N messages | `produce_messages(producer, topic, n, partition, timestamp).await` (from `utils::*`) | @@ -315,8 +316,12 @@ failure you didn't introduce. - **test**: the integration suite. Fans out across `KAFKA_VERSION = 3.7, 3.8, 3.9, 4.0`. Each row resolves to a specific `apache/kafka:` via `resolve_kafka_image_tag` and runs - `cargo test`. Rows run sequentially (`max-parallel: 1`) because they - share an Actions runner and each spawns its own Docker container. + `cargo test --features zstd`. The `zstd` feature is on so the + compression round-trip test for zstd in `tests/future_producer.rs` + actually links the codec (rdkafka-sys passes `--disable-zstd` to + librdkafka by default). Rows run sequentially (`max-parallel: 1`) + because they share an Actions runner and each spawns its own Docker + container. - **runtime-examples**: smoke-tests `examples/runtime_smol.rs` and `examples/runtime_async_std.rs` against a pinned `apache/kafka:4.0.2` service container. The integration suite covers the tokio path via diff --git a/tests/future_producer.rs b/tests/future_producer.rs index 3a0699839..dcb3a7cc8 100644 --- a/tests/future_producer.rs +++ b/tests/future_producer.rs @@ -2,11 +2,13 @@ use std::time::{Duration, Instant}; +use futures::future; use futures::stream::{FuturesUnordered, StreamExt}; use rdkafka::admin::AdminOptions; use rdkafka::client::DefaultClientContext; use rdkafka::config::ClientConfig; +use rdkafka::consumer::Consumer; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; use rdkafka::message::{Header, Headers, Message, OwnedHeaders}; use rdkafka::producer::{FutureProducer, FutureRecord, Producer}; @@ -192,6 +194,114 @@ async fn test_future_producer_send_fail() { } } +async fn run_compression_round_trip(codec: &str) { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic(&format!("test_compression_{}", codec)); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer_with_overrides( + &kafka_context.bootstrap_servers, + &[("compression.type", codec), ("linger.ms", "20")], + ) + .await + .expect("could not create future producer"); + + const N: usize = 64; + let payload = "rust-rdkafka compression round trip ".repeat(8); + let keys: Vec = (0..N).map(|i| format!("k{}", i)).collect(); + let values: Vec = (0..N).map(|i| format!("{}:{}", i, payload)).collect(); + let mut futures = Vec::with_capacity(N); + for i in 0..N { + futures.push( + producer.send( + FutureRecord::to(&topic_name) + .partition(0) + .key(&keys[i]) + .payload(&values[i]), + Duration::from_secs(10), + ), + ); + } + let mut expected = std::collections::HashMap::with_capacity(N); + for (i, future) in futures.into_iter().enumerate() { + let delivered = future.await.unwrap_or_else(|(e, _)| { + panic!("delivery failed for codec {} message {}: {}", codec, i, e) + }); + expected.insert(delivered.offset, (keys[i].clone(), values[i].clone())); + } + producer + .flush(Timeout::After(Duration::from_secs(10))) + .unwrap(); + + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); + consumer.subscribe(&[topic_name.as_str()]).unwrap(); + + let mut seen = 0usize; + consumer + .stream() + .take(N) + .for_each(|message| { + let m = message.expect("error receiving message"); + let (expected_key, expected_value) = expected + .remove(&m.offset()) + .unwrap_or_else(|| panic!("unexpected offset {} for codec {}", m.offset(), codec)); + assert_eq!(m.key_view::().unwrap().unwrap(), expected_key); + assert_eq!(m.payload_view::().unwrap().unwrap(), expected_value); + seen += 1; + future::ready(()) + }) + .await; + assert_eq!(seen, N, "codec {} did not yield all messages", codec); + assert!( + expected.is_empty(), + "codec {} left {} unmatched offsets", + codec, + expected.len() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_future_producer_compression_gzip() { + run_compression_round_trip("gzip").await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_future_producer_compression_snappy() { + run_compression_round_trip("snappy").await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_future_producer_compression_lz4() { + run_compression_round_trip("lz4").await; +} + +// librdkafka is built with `--disable-zstd` unless the `zstd` Cargo feature is +// enabled (see rdkafka-sys/build.rs), so this test can only run when that +// feature is on. CI exercises it by passing `--features zstd` to the test job. +#[cfg(feature = "zstd")] +#[tokio::test(flavor = "multi_thread")] +async fn test_future_producer_compression_zstd() { + run_compression_round_trip("zstd").await; +} + #[tokio::test] async fn test_future_undelivered() { let delivery_future = { diff --git a/tests/utils/producer/future_producer.rs b/tests/utils/producer/future_producer.rs index 8d4891ad7..712ddb94d 100644 --- a/tests/utils/producer/future_producer.rs +++ b/tests/utils/producer/future_producer.rs @@ -4,8 +4,18 @@ use rdkafka::producer::FutureProducer; use rdkafka::ClientConfig; pub async fn create_producer(bootstrap_servers: &str) -> anyhow::Result { + create_producer_with_overrides(bootstrap_servers, &[]).await +} + +pub async fn create_producer_with_overrides( + bootstrap_servers: &str, + config_overrides: &[(&str, &str)], +) -> anyhow::Result { let mut producer_client_config = ClientConfig::default(); producer_client_config.set("bootstrap.servers", bootstrap_servers); + for (key, value) in config_overrides { + producer_client_config.set(*key, *value); + } let future_producer = FutureProducer::from_config(&producer_client_config) .context("couldn't create producer client")?; Ok(future_producer) From 2c90c8eb3b89620d684acded0239bd98abec1003 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:17:06 -0500 Subject: [PATCH 41/56] test: cover idempotent FutureProducer round-trip Adds `test_future_producer_idempotence` in tests/future_producer.rs. It opens a FutureProducer with `enable.idempotence=true`, sends 1000 distinct payloads in two batches separated by a `flush()`, then consumes them back through a stream consumer and asserts the result has exactly 1000 distinct payloads on offsets 0..999. That is the exactly-once contract librdkafka promises for the idempotent producer, and the binding has to wire the implicit `acks=all` / retries / in-flight cap / sequence-number tracking correctly for the test to pass. A true "forced reconnect midway" requires either a TCP proxy or a privileged in-process disconnect; both are out of scope for the testcontainer setup. The flush boundary is a stand-in that exercises the producer's recovery from a fully-drained pipeline. --- tests/future_producer.rs | 125 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/tests/future_producer.rs b/tests/future_producer.rs index dcb3a7cc8..04ea10e54 100644 --- a/tests/future_producer.rs +++ b/tests/future_producer.rs @@ -302,6 +302,131 @@ async fn test_future_producer_compression_zstd() { run_compression_round_trip("zstd").await; } +// Enables the idempotent producer and produces in two batches separated by a +// flush, so the second batch starts after the first has fully drained. If the +// PID/epoch tracking that librdkafka enables under `enable.idempotence=true` +// (acks=all, retries, in-flight bound, sequence numbers) regresses in the +// binding, the consumer side will see duplicate or missing offsets / payloads. +// +// A true "forced reconnect midway" requires either a proxy or a privileged +// in-process disconnect; both are out of scope for the testcontainer setup, +// and aggressive `connections.max.idle.ms` produces MessageTimedOut errors +// rather than testing idempotence. The flush boundary is a cheap stand-in +// that exercises the producer's recovery from a fully-drained pipeline. +#[tokio::test(flavor = "multi_thread")] +async fn test_future_producer_idempotence() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_future_producer_idempotence"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer_with_overrides( + &kafka_context.bootstrap_servers, + &[ + ("enable.idempotence", "true"), + ("message.timeout.ms", "30000"), + ], + ) + .await + .expect("could not create idempotent future producer"); + + const N: usize = 1000; + const HALF: usize = N / 2; + let payloads: Vec = (0..N).map(|i| format!("idempotent-{:04}", i)).collect(); + + let mut delivery_futures = Vec::with_capacity(HALF); + for payload in &payloads[..HALF] { + delivery_futures.push( + producer.send( + FutureRecord::<(), str>::to(&topic_name) + .partition(0) + .payload(payload), + Duration::from_secs(30), + ), + ); + } + for (i, fut) in delivery_futures.into_iter().enumerate() { + fut.await + .unwrap_or_else(|(e, _)| panic!("first-half delivery {} failed: {}", i, e)); + } + + producer + .flush(Timeout::After(Duration::from_secs(30))) + .unwrap(); + + let mut delivery_futures = Vec::with_capacity(N - HALF); + for payload in &payloads[HALF..] { + delivery_futures.push( + producer.send( + FutureRecord::<(), str>::to(&topic_name) + .partition(0) + .payload(payload), + Duration::from_secs(30), + ), + ); + } + for (i, fut) in delivery_futures.into_iter().enumerate() { + fut.await + .unwrap_or_else(|(e, _)| panic!("second-half delivery {} failed: {}", i, e)); + } + producer + .flush(Timeout::After(Duration::from_secs(30))) + .unwrap(); + + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); + consumer.subscribe(&[topic_name.as_str()]).unwrap(); + + let mut seen_offsets = std::collections::BTreeSet::new(); + let mut seen_payloads = std::collections::HashSet::new(); + consumer + .stream() + .take(N) + .for_each(|message| { + let m = message.expect("error receiving message"); + assert!( + seen_offsets.insert(m.offset()), + "duplicate offset {} delivered", + m.offset() + ); + let payload = m.payload_view::().unwrap().unwrap().to_string(); + assert!( + seen_payloads.insert(payload.clone()), + "duplicate payload {} delivered", + payload + ); + future::ready(()) + }) + .await; + assert_eq!(seen_offsets.len(), N); + assert_eq!(*seen_offsets.iter().next().unwrap(), 0); + assert_eq!(*seen_offsets.iter().next_back().unwrap(), (N as i64) - 1); + assert_eq!(seen_payloads.len(), N); + for payload in &payloads { + assert!( + seen_payloads.contains(payload), + "missing payload {}", + payload + ); + } +} + #[tokio::test] async fn test_future_undelivered() { let delivery_future = { From aa9a36b7c3f318cb83656cdc1a6cacbf58a5df84 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:19:50 -0500 Subject: [PATCH 42/56] test: cover ConsumerContext pre/post rebalance callbacks Adds `test_consumer_rebalance_callbacks` in tests/base_consumer.rs. The test wires a `RecordingRebalanceContext` that captures every pre_rebalance and post_rebalance event with the affected partitions, creates a topic with two partitions, drives consumer 1 until both partitions are assigned, then attaches consumer 2 to the same group and drives both consumers until the group converges to one partition each. Assertions then verify that consumer 1 saw a post-rebalance Assign event with both partitions followed by a Revoke (when consumer 2 joined), and that consumer 2 saw a post-rebalance Assign event with exactly one partition. A binding regression that drops one of those callbacks would cause this test to time out or fail the kind/count check. --- tests/base_consumer.rs | 206 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 204 insertions(+), 2 deletions(-) diff --git a/tests/base_consumer.rs b/tests/base_consumer.rs index 4035f4a19..484675d29 100644 --- a/tests/base_consumer.rs +++ b/tests/base_consumer.rs @@ -3,12 +3,13 @@ use std::collections::HashMap; use std::convert::TryInto; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; use rdkafka::admin::AdminOptions; -use rdkafka::consumer::{BaseConsumer, Consumer, ConsumerContext}; +use rdkafka::client::ClientContext; +use rdkafka::consumer::{BaseConsumer, Consumer, ConsumerContext, Rebalance}; use rdkafka::error::{KafkaError, RDKafkaErrorCode}; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::{current_time_millis, Timeout}; @@ -685,3 +686,204 @@ async fn test_invalid_consumer_position() { Err(KafkaError::MetadataFetch(RDKafkaErrorCode::UnknownGroup)) ); } + +#[derive(Clone, Debug, PartialEq, Eq)] +enum RebalanceEventKind { + Assign, + Revoke, + Error, +} + +#[derive(Clone, Debug)] +struct RebalanceEvent { + kind: RebalanceEventKind, + partitions: Vec<(String, i32)>, +} + +#[derive(Clone)] +struct RecordingRebalanceContext { + pre: Arc>>, + post: Arc>>, +} + +impl RecordingRebalanceContext { + fn new() -> Self { + Self { + pre: Arc::new(Mutex::new(Vec::new())), + post: Arc::new(Mutex::new(Vec::new())), + } + } + + fn drain(&self) -> (Vec, Vec) { + let pre = self.pre.lock().unwrap().clone(); + let post = self.post.lock().unwrap().clone(); + (pre, post) + } +} + +fn record_rebalance(rebalance: &Rebalance) -> RebalanceEvent { + match rebalance { + Rebalance::Assign(tpl) => RebalanceEvent { + kind: RebalanceEventKind::Assign, + partitions: tpl + .elements() + .iter() + .map(|e| (e.topic().to_string(), e.partition())) + .collect(), + }, + Rebalance::Revoke(tpl) => RebalanceEvent { + kind: RebalanceEventKind::Revoke, + partitions: tpl + .elements() + .iter() + .map(|e| (e.topic().to_string(), e.partition())) + .collect(), + }, + Rebalance::Error(_) => RebalanceEvent { + kind: RebalanceEventKind::Error, + partitions: Vec::new(), + }, + } +} + +impl ClientContext for RecordingRebalanceContext {} + +impl ConsumerContext for RecordingRebalanceContext { + fn pre_rebalance(&self, _: &BaseConsumer, rebalance: &Rebalance) { + self.pre.lock().unwrap().push(record_rebalance(rebalance)); + } + + fn post_rebalance(&self, _: &BaseConsumer, rebalance: &Rebalance) { + self.post.lock().unwrap().push(record_rebalance(rebalance)); + } +} + +fn build_recording_consumer( + bootstrap_servers: &str, + group_id: &str, +) -> BaseConsumer { + let mut config = ClientConfig::new(); + config + .set("group.id", group_id) + .set("bootstrap.servers", bootstrap_servers) + .set("enable.partition.eof", "false") + .set("session.timeout.ms", "6000") + .set("enable.auto.commit", "false") + .set("auto.offset.reset", "earliest"); + config + .create_with_context::>( + RecordingRebalanceContext::new(), + ) + .expect("could not create recording base consumer") +} + +#[tokio::test] +async fn test_consumer_rebalance_callbacks() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_consumer_rebalance_callbacks"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(2)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let group = rand_test_group(); + + let consumer1 = build_recording_consumer(&kafka_context.bootstrap_servers, &group); + let context1 = consumer1.context().clone(); + consumer1.subscribe(&[topic_name.as_str()]).unwrap(); + + let deadline = Instant::now() + Duration::from_secs(20); + loop { + consumer1.poll(Duration::from_millis(200)); + let assignment = consumer1.assignment().unwrap(); + if assignment.count() == 2 { + break; + } + if Instant::now() > deadline { + panic!( + "consumer1 never got both partitions; assignment count {}", + assignment.count() + ); + } + } + + let consumer2 = build_recording_consumer(&kafka_context.bootstrap_servers, &group); + let context2 = consumer2.context().clone(); + consumer2.subscribe(&[topic_name.as_str()]).unwrap(); + + let deadline = Instant::now() + Duration::from_secs(30); + loop { + consumer1.poll(Duration::from_millis(200)); + consumer2.poll(Duration::from_millis(200)); + let a1 = consumer1.assignment().unwrap().count(); + let a2 = consumer2.assignment().unwrap().count(); + if a1 == 1 && a2 == 1 { + break; + } + if Instant::now() > deadline { + panic!( + "rebalance did not converge to one partition per consumer: c1={}, c2={}", + a1, a2 + ); + } + } + + let (pre1, post1) = context1.drain(); + let (pre2, post2) = context2.drain(); + + assert!( + pre1.iter().any(|e| e.kind == RebalanceEventKind::Assign), + "consumer1 never observed a pre-rebalance Assign event: {:?}", + pre1 + ); + assert!( + post1.iter().any(|e| e.kind == RebalanceEventKind::Assign), + "consumer1 never observed a post-rebalance Assign event: {:?}", + post1 + ); + let first_assign1 = post1 + .iter() + .find(|e| e.kind == RebalanceEventKind::Assign) + .expect("missing initial assign on consumer1"); + assert_eq!( + first_assign1.partitions.len(), + 2, + "consumer1's first post-rebalance assign should hold both partitions, got {:?}", + first_assign1.partitions + ); + for (topic, _) in &first_assign1.partitions { + assert_eq!(topic, &topic_name); + } + assert!( + post1.iter().any(|e| e.kind == RebalanceEventKind::Revoke), + "consumer1 never observed a post-rebalance Revoke event after consumer2 joined: {:?}", + post1 + ); + + assert!( + pre2.iter().any(|e| e.kind == RebalanceEventKind::Assign), + "consumer2 never observed a pre-rebalance Assign event: {:?}", + pre2 + ); + let assign2 = post2 + .iter() + .find(|e| e.kind == RebalanceEventKind::Assign) + .expect("consumer2 never observed a post-rebalance Assign event"); + assert_eq!( + assign2.partitions.len(), + 1, + "consumer2 should have been assigned exactly one partition, got {:?}", + assign2.partitions + ); + assert_eq!(assign2.partitions[0].0, topic_name); +} From f5989c406ad3ca9d45dbe107dc3d600f6e4832e6 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:23:05 -0500 Subject: [PATCH 43/56] test: cover send_offsets_to_transaction on a consume-transform-produce loop `test_transaction_commit` already calls `send_offsets_to_transaction`, but it produces a fixed `A` payload regardless of what was consumed and only inspects the consumer's committed offset at the end. Adds `test_transaction_send_offsets_consume_transform_produce` in tests/transactions.rs that does a real consume-transform-produce loop in two transactions: each transaction reads a batch, transforms each payload, produces the transformed payload to the output topic, ties the consumer offset advance to the transaction via `send_offsets_to_transaction`, and commits. Assertions then check that the consumer's committed offset equals the total number of consumed messages (it advanced exactly once per batch, in lockstep with the transaction) and that the output topic, read with `isolation.level=read_committed`, contains the transformed payloads in order. A binding regression on `send_offsets_to_transaction` would either skip an offset advance or write the wrong payload through the transactional producer. --- tests/transactions.rs | 140 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/tests/transactions.rs b/tests/transactions.rs index a3a0fe791..e42c4f64a 100644 --- a/tests/transactions.rs +++ b/tests/transactions.rs @@ -10,6 +10,7 @@ use rdkafka::config::ClientConfig; use rdkafka::config::RDKafkaLogLevel; use rdkafka::consumer::{BaseConsumer, CommitMode, Consumer}; use rdkafka::error::KafkaError; +use rdkafka::message::Message; use rdkafka::producer::{BaseProducer, BaseRecord, Producer}; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::Timeout; @@ -309,3 +310,142 @@ async fn test_transaction_commit() -> Result<(), Box> { Ok(()) } + +// `test_transaction_commit` already calls `send_offsets_to_transaction`, but +// it produces a fixed "A" payload regardless of what was consumed and only +// looks at offsets at the end. This test does a full consume-transform-produce +// loop in two transactions: each transaction reads a batch from the input +// topic, transforms each payload, produces the transformed payload to the +// output topic, ties the consumer's offset advance to the transaction with +// `send_offsets_to_transaction`, and commits. The assertion shape catches a +// binding regression where the transformed payloads diverge from the input or +// where `send_offsets_to_transaction` fails to advance the consumer-side +// committed offset in step with the transaction. +#[tokio::test] +async fn test_transaction_send_offsets_consume_transform_produce() -> Result<(), Box> { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let consume_topic = rand_test_topic("test_txn_ctp_in"); + let produce_topic = rand_test_topic("test_txn_ctp_out"); + + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&consume_topic, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create consume topic"); + admin_client + .create_topics( + &admin::new_topic_vec(&produce_topic, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create produce topic"); + + let future_producer = + producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("could not create future producer"); + const BATCHES: usize = 2; + const PER_BATCH: usize = 5; + const TOTAL: usize = BATCHES * PER_BATCH; + let _ = produce_messages_to_partition(&future_producer, &consume_topic, TOTAL, 0).await; + + let consumer = create_consumer(&kafka_context, None).await?; + consumer.subscribe(&[&consume_topic])?; + + let producer = create_producer(&kafka_context)?; + producer.init_transactions(Timeout::Never)?; + + let mut expected_outputs: Vec = Vec::with_capacity(TOTAL); + let mut consumed = 0usize; + for batch in 0..BATCHES { + producer.begin_transaction()?; + + let mut batch_payloads: Vec = Vec::with_capacity(PER_BATCH); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while batch_payloads.len() < PER_BATCH { + if std::time::Instant::now() > deadline { + panic!( + "consumer poll timed out in batch {} after {} messages", + batch, + batch_payloads.len() + ); + } + let Some(message) = consumer.poll(Duration::from_secs(2)) else { + continue; + }; + let message = message?; + let payload = message.payload_view::().unwrap().unwrap().to_string(); + batch_payloads.push(payload); + consumed += 1; + } + + for payload in &batch_payloads { + let transformed = format!("transformed:{}", payload); + expected_outputs.push(transformed.clone()); + producer + .send( + BaseRecord::to(&produce_topic) + .payload(&transformed) + .key("k") + .partition(0), + ) + .map_err(|(e, _)| e)?; + } + + let cgm = consumer.group_metadata().unwrap(); + let mut txn_tpl = TopicPartitionList::new(); + txn_tpl.add_partition_offset(&consume_topic, 0, Offset::Offset(consumed as i64))?; + producer.send_offsets_to_transaction(&txn_tpl, &cgm, Timeout::Never)?; + + producer.flush(Duration::from_secs(20))?; + producer.commit_transaction(Duration::from_secs(20))?; + } + + let committed = consumer.committed(Timeout::Never)?; + assert_eq!( + committed + .find_partition(&consume_topic, 0) + .unwrap() + .offset(), + Offset::Offset(TOTAL as i64), + "consumer-side committed offset must equal the number of consumed messages", + ); + + let output_consumer = create_consumer( + &kafka_context, + Some(&[ + ("isolation.level", "read_committed"), + ("enable.partition.eof", "true"), + ]), + ) + .await?; + let mut tpl = TopicPartitionList::new(); + tpl.add_partition(&produce_topic, 0); + output_consumer.assign(&tpl)?; + let mut output_payloads = Vec::with_capacity(TOTAL); + for message in output_consumer.iter() { + match message { + Ok(m) => { + let payload = m.payload_view::().unwrap().unwrap().to_string(); + output_payloads.push(payload); + } + Err(KafkaError::PartitionEOF(_)) => break, + Err(e) => return Err(e.into()), + } + } + assert_eq!( + output_payloads, expected_outputs, + "output topic payloads must match the transformed inputs in order", + ); + + Ok(()) +} From 9e9e38bdfca6e74cdf7db8d0bbf44b76a7ca5ab0 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:24:51 -0500 Subject: [PATCH 44/56] test: assert MessageSizeTooLarge on oversized BaseProducer send Adds `test_base_producer_message_too_large` in tests/base_producer.rs. The test configures a BaseProducer with `message.max.bytes=1024`, calls `send` with a 4096-byte payload, and asserts that the synchronous return is `Err((KafkaError::MessageProduction(RDKafkaErrorCode::MessageSizeTooLarge), _))`. It then sends a baseline small payload and flushes, and asserts the collecting context observed exactly that one delivery, so a binding regression that swallowed the oversized error or silently queued the oversized message would surface as either the wrong error variant or an unexpected second delivery. --- tests/base_producer.rs | 71 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/base_producer.rs b/tests/base_producer.rs index 541e5edc7..c4750c4d4 100644 --- a/tests/base_producer.rs +++ b/tests/base_producer.rs @@ -623,6 +623,77 @@ async fn test_custom_partitioner_base_producer() { } } +// librdkafka rejects payloads larger than `message.max.bytes` synchronously +// inside `rd_kafka_produce`, surfacing +// `MessageProduction(RDKafkaErrorCode::MessageSizeTooLarge)` from +// `BaseProducer::send`. A binding regression that swallows or remaps that +// error would cause the producer to hang on the eventual flush or silently +// drop the message; this test asserts the synchronous error shape so that +// regression is loud. +#[tokio::test(flavor = "multi_thread")] +async fn test_base_producer_message_too_large() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_base_producer_message_too_large"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let context = CollectingContext::new(); + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + context.clone(), + &[("message.max.bytes", "1024")], + ) + .expect("failed to create base producer"); + + let oversized = vec![b'x'; 4096]; + let result = producer.send( + BaseRecord::with_opaque_to(&topic_name, 0usize) + .payload(&oversized) + .key("k"), + ); + + match result { + Err((KafkaError::MessageProduction(RDKafkaErrorCode::MessageSizeTooLarge), _record)) => {} + Err((other, _)) => panic!("unexpected error variant: {:?}", other), + Ok(()) => panic!("send unexpectedly succeeded for an oversized payload"), + } + + let small_record = BaseRecord::with_opaque_to(&topic_name, 1usize) + .payload(b"ok" as &[u8]) + .key("k"); + producer + .send(small_record) + .expect("baseline small send should succeed"); + producer.flush(Duration::from_secs(10)).unwrap(); + + let delivered: Vec<_> = context + .results + .lock() + .unwrap() + .iter() + .map(|(_, err, id)| (err.clone(), *id)) + .collect(); + assert_eq!( + delivered.len(), + 1, + "only the baseline small payload should reach the delivery callback: {:?}", + delivered + ); + assert_eq!(delivered[0], (None, 1)); +} + #[tokio::test(flavor = "multi_thread")] async fn test_custom_partitioner_threaded_producer() { init_test_logger(); From f525dcdd9868c11f073fba2f4b10e8b5a2996ba3 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:26:09 -0500 Subject: [PATCH 45/56] test: assert default partitioner maps each key to one partition Adds `test_future_producer_default_partitioner_is_deterministic` in tests/future_producer.rs. With a six-partition topic and a four-key working set, the test produces 16 copies of each key (no explicit partition) and verifies every copy of a given key landed on the same partition. It also asserts the four keys span at least two distinct partitions, so a regression that collapsed everything onto partition 0 would also fail. A binding break that dropped or rewrote the key on the way into librdkafka would scatter the same key across partitions and surface here. --- tests/future_producer.rs | 66 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/future_producer.rs b/tests/future_producer.rs index 04ea10e54..699d13609 100644 --- a/tests/future_producer.rs +++ b/tests/future_producer.rs @@ -427,6 +427,72 @@ async fn test_future_producer_idempotence() { } } +// librdkafka's default `consistent_random` partitioner hashes keyed records +// to a partition with CRC32. A binding regression that drops or rewrites the +// key on the way into librdkafka would cause the same key to map to +// different partitions across sends; this test produces 16 copies of each +// key across 4 keys and asserts every copy of each key landed on exactly one +// partition. +#[tokio::test(flavor = "multi_thread")] +async fn test_future_producer_default_partitioner_is_deterministic() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_future_producer_default_partitioner"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(6)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("could not create future producer"); + + let keys = ["alpha", "beta", "gamma", "delta"]; + const COPIES: usize = 16; + + let mut per_key: std::collections::HashMap<&str, std::collections::HashSet> = + std::collections::HashMap::new(); + for (k_idx, key) in keys.iter().enumerate() { + for c in 0..COPIES { + let payload = format!("payload-{}-{}", k_idx, c); + let delivered = producer + .send( + FutureRecord::to(&topic_name).key(*key).payload(&payload), + Duration::from_secs(10), + ) + .await + .unwrap_or_else(|(e, _)| panic!("delivery failed for key {}: {}", key, e)); + per_key.entry(*key).or_default().insert(delivered.partition); + } + } + + for key in keys { + let partitions = per_key.get(key).expect("missing deliveries for key"); + assert_eq!( + partitions.len(), + 1, + "key {} mapped to multiple partitions: {:?}", + key, + partitions + ); + } + let chosen: std::collections::HashSet = per_key.values().flatten().copied().collect(); + assert!( + chosen.len() >= 2, + "with four keys over six partitions we should see at least two distinct partitions, got {:?}", + chosen + ); +} + #[tokio::test] async fn test_future_undelivered() { let delivery_future = { From 0a056faf270b4173b893cad2a1ef3909e18f2084 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:27:10 -0500 Subject: [PATCH 46/56] test: assert StreamConsumer surfaces produced headers verbatim Adds `test_consumer_reads_message_headers` in tests/stream_consumers.rs. The existing `test_base_producer_headers` only validates the produce side via the delivery callback; this test produces a message with a mixed header set (a byte-valued header, a str-valued header, an empty header, and an explicitly null header), consumes it through a StreamConsumer, and verifies that `BorrowedMessage::headers` exposes each header in order with the matching key, value type, and value bytes. A binding regression in `BorrowedMessage::headers` or the underlying headers FFI conversion would surface here. --- tests/stream_consumers.rs | 132 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/tests/stream_consumers.rs b/tests/stream_consumers.rs index dab35d16b..dbdf0eb28 100644 --- a/tests/stream_consumers.rs +++ b/tests/stream_consumers.rs @@ -13,6 +13,8 @@ use tokio::time::{self, Duration}; use rdkafka::admin::AdminOptions; use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer}; use rdkafka::error::KafkaError; +use rdkafka::message::{Header, Headers, OwnedHeaders}; +use rdkafka::producer::FutureRecord; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; use rdkafka::util::current_time_millis; use rdkafka::{Message, Timestamp}; @@ -934,3 +936,133 @@ async fn test_consume_partition_order() { worker.await.unwrap(); } } + +// `test_base_producer_headers` already covers the produce side via the +// delivery callback. This test covers the consume side: produce a message +// with a mixed set of headers (str-valued, byte-valued, empty, and explicitly +// null), consume it through a StreamConsumer, and verify +// `BorrowedMessage::headers` exposes each header in order with the correct +// key, value type, and value bytes. A binding regression in the headers +// FFI conversion path would surface here. +#[tokio::test(flavor = "multi_thread")] +async fn test_consumer_reads_message_headers() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_consumer_reads_message_headers"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("could not create future producer"); + + let headers = OwnedHeaders::new() + .insert(Header { + key: "h-bytes", + value: Some(&[0u8, 1, 2, 3][..]), + }) + .insert(Header { + key: "h-str", + value: Some("v-str"), + }) + .insert(Header { + key: "h-empty", + value: Some(&[][..]), + }) + .insert::>(Header { + key: "h-null", + value: None, + }); + + producer + .send( + FutureRecord::to(&topic_name) + .partition(0) + .key("k") + .payload("p") + .headers(headers), + Duration::from_secs(10), + ) + .await + .unwrap_or_else(|(e, _)| panic!("delivery failed: {}", e)); + + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); + consumer.subscribe(&[topic_name.as_str()]).unwrap(); + + let message = time::timeout(Duration::from_secs(15), consumer.stream().next()) + .await + .expect("timed out waiting for consumed message") + .expect("stream ended unexpectedly") + .expect("error receiving message"); + + let received = message + .headers() + .expect("consumed message should expose headers"); + assert_eq!(received.count(), 4); + assert_eq!( + received.get(0), + Header { + key: "h-bytes", + value: Some(&[0u8, 1, 2, 3][..]), + } + ); + assert_eq!( + received.get_as::(1), + Ok(Header { + key: "h-str", + value: Some("v-str"), + }) + ); + assert_eq!( + received.get_as::<[u8]>(2), + Ok(Header { + key: "h-empty", + value: Some(&[][..]), + }) + ); + assert_eq!( + received.get_as::<[u8]>(3), + Ok(Header { + key: "h-null", + value: None, + }) + ); + let collected: Vec<_> = received.iter().collect(); + assert_eq!( + collected, + vec![ + Header { + key: "h-bytes", + value: Some(&[0u8, 1, 2, 3][..]), + }, + Header { + key: "h-str", + value: Some(b"v-str" as &[u8]), + }, + Header { + key: "h-empty", + value: Some(&[][..]), + }, + Header { + key: "h-null", + value: None, + }, + ], + ); +} From ea2d33ce3209d314922614623cdf12e6c620efe2 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:27:55 -0500 Subject: [PATCH 47/56] test: assert Producer::flush drains in-flight before returning Adds `test_base_producer_flush_drains_inflight` in tests/base_producer.rs. The test queues 200 records on a BaseProducer with `linger.ms=100` so that they stay buffered, asserts that `in_flight_count()` is non-zero immediately after the send loop, then calls `flush(20s)` and asserts both that `in_flight_count()` drops to zero and that the collecting context received exactly 200 successful delivery callbacks. A binding regression that returned from `flush` before the queue had drained, or misreported the in-flight count, would surface either as a non-zero counter or a short delivery callback list. --- tests/base_producer.rs | 71 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/base_producer.rs b/tests/base_producer.rs index c4750c4d4..d637ccb49 100644 --- a/tests/base_producer.rs +++ b/tests/base_producer.rs @@ -623,6 +623,77 @@ async fn test_custom_partitioner_base_producer() { } } +// Asserts the synchronous contract of `Producer::flush`: when `flush(timeout)` +// returns successfully, every previously-queued record has reached its +// delivery callback and the in-flight counter is zero. A binding regression +// that returned early from `flush` (or surfaced the wrong in-flight value) +// would let a caller drop the producer while messages were still buffered. +#[tokio::test(flavor = "multi_thread")] +async fn test_base_producer_flush_drains_inflight() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_base_producer_flush_drains_inflight"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let context = CollectingContext::new(); + // `linger.ms=100` buffers records for up to 100ms before sending, so the + // in-flight count is non-zero immediately after the loop and `flush` has + // real work to drain. + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + context.clone(), + &[("linger.ms", "100")], + ) + .expect("failed to create base producer"); + + const N: usize = 200; + for id in 0..N { + producer + .send( + BaseRecord::with_opaque_to(&topic_name, id) + .payload("payload") + .key("key"), + ) + .expect("send failed"); + } + assert!( + producer.in_flight_count() > 0, + "expected non-zero in-flight count after queuing {} records, got 0", + N + ); + + producer + .flush(Duration::from_secs(20)) + .expect("flush returned an error"); + + assert_eq!( + producer.in_flight_count(), + 0, + "in-flight count must be zero after a successful flush", + ); + let delivered = context.results.lock().unwrap(); + assert_eq!( + delivered.len(), + N, + "every queued record should have hit the delivery callback by the time flush returns", + ); + for (_, error, _) in delivered.iter() { + assert!(error.is_none(), "unexpected delivery error: {:?}", error); + } +} + // librdkafka rejects payloads larger than `message.max.bytes` synchronously // inside `rd_kafka_produce`, surfacing // `MessageProduction(RDKafkaErrorCode::MessageSizeTooLarge)` from From 26c8b96f28d733e9d174fbe68bf7e50452738b56 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:29:24 -0500 Subject: [PATCH 48/56] test: assert offsets_for_times returns first offset at-or-after query Adds `test_consumer_offsets_for_times_first_at_or_after` in tests/stream_consumers.rs. The existing `test_produce_consume_with_timestamp` exercises `offsets_for_timestamp` but only with two distinct timestamp values; this test produces 20 messages with a strictly monotonic per-message timestamp and then queries at three points: an exact-match timestamp, a midpoint between two adjacent message timestamps, and a timestamp past the high watermark. The midpoint case asserts the returned offset is the first message at-or-after the query (offset 8 for a query between offsets 7 and 8); the past-watermark case asserts `Offset::End`. A binding regression that returned the nearest-neighbour offset or that mangled the timestamp/offset TPL serialization would fail one of those assertions. --- tests/stream_consumers.rs | 103 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/tests/stream_consumers.rs b/tests/stream_consumers.rs index dbdf0eb28..f35e9c308 100644 --- a/tests/stream_consumers.rs +++ b/tests/stream_consumers.rs @@ -937,6 +937,109 @@ async fn test_consume_partition_order() { } } +// `test_produce_consume_with_timestamp` already calls `offsets_for_timestamp`, +// but only with two distinct timestamp values. This test produces a strictly +// monotonic sequence of distinct timestamps and queries at a midpoint, then +// verifies the returned offset is the first message at-or-after the query +// timestamp. A binding regression in `offsets_for_times` (or its `tpl` +// serialization) that returned the nearest neighbour, or the first ever +// offset, would fail this assertion. +#[tokio::test(flavor = "multi_thread")] +async fn test_consumer_offsets_for_times_first_at_or_after() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_consumer_offsets_for_times_first_at_or_after"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("could not create future producer"); + + const N: i64 = 20; + const BASE_TS: i64 = 1_700_000_000_000; + const STEP: i64 = 100; + for i in 0..N { + let ts = BASE_TS + i * STEP; + let payload = format!("ts-{}", ts); + producer + .send( + FutureRecord::to(&topic_name) + .partition(0) + .key("k") + .payload(&payload) + .timestamp(ts), + Duration::from_secs(10), + ) + .await + .unwrap_or_else(|(e, _)| panic!("delivery failed: {}", e)); + } + + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); + let mut tpl = TopicPartitionList::new(); + tpl.add_partition_offset(&topic_name, 0, Offset::Beginning) + .unwrap(); + consumer.assign(&tpl).unwrap(); + + let exact_query_ts = BASE_TS + 7 * STEP; + let exact_tpl = consumer + .offsets_for_timestamp(exact_query_ts, Duration::from_secs(10)) + .expect("offsets_for_timestamp failed for exact timestamp"); + let exact_offset = exact_tpl + .find_partition(&topic_name, 0) + .expect("missing partition entry") + .offset(); + assert_eq!( + exact_offset, + Offset::Offset(7), + "exact-timestamp query should return the matching offset", + ); + + let midpoint_query_ts = BASE_TS + 7 * STEP + STEP / 2; + let midpoint_tpl = consumer + .offsets_for_timestamp(midpoint_query_ts, Duration::from_secs(10)) + .expect("offsets_for_timestamp failed for midpoint timestamp"); + let midpoint_offset = midpoint_tpl + .find_partition(&topic_name, 0) + .expect("missing partition entry") + .offset(); + assert_eq!( + midpoint_offset, + Offset::Offset(8), + "midpoint-timestamp query should return the first offset at-or-after the query", + ); + + let after_query_ts = BASE_TS + (N + 10) * STEP; + let after_tpl = consumer + .offsets_for_timestamp(after_query_ts, Duration::from_secs(10)) + .expect("offsets_for_timestamp failed for after-end timestamp"); + let after_offset = after_tpl + .find_partition(&topic_name, 0) + .expect("missing partition entry") + .offset(); + assert_eq!( + after_offset, + Offset::End, + "timestamp past the high watermark should return Offset::End", + ); +} + // `test_base_producer_headers` already covers the produce side via the // delivery callback. This test covers the consume side: produce a message // with a mixed set of headers (str-valued, byte-valued, empty, and explicitly From 04872b77e21d1f3cb7ba3970b8aa64d5dc60dc93 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:30:24 -0500 Subject: [PATCH 49/56] test: assert regex subscription assigns only matching topics Adds `test_consumer_regex_subscription_matches_only_prefixed` in tests/stream_consumers.rs. The test creates two topics that match a unique `^.*` pattern and one topic that does not, produces a message into each, subscribes a StreamConsumer to the regex, and drives the stream until the consumer's assignment stabilises. It then asserts both that the assignment is exactly the two matching topics and that the non-matching topic was never delivered to the stream. A binding regression that dropped the leading `^` or that crossed up topic ownership during the regex resolve would surface here. --- tests/stream_consumers.rs | 98 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/stream_consumers.rs b/tests/stream_consumers.rs index f35e9c308..d17cfa5f6 100644 --- a/tests/stream_consumers.rs +++ b/tests/stream_consumers.rs @@ -937,6 +937,104 @@ async fn test_consume_partition_order() { } } +// librdkafka treats subscription strings beginning with `^` as a regex +// pattern and resolves them against the broker's topic metadata on every +// metadata refresh. This test creates two topics that match a unique +// `^.*` pattern and one topic that does not, subscribes the consumer +// to the regex, drives the consumer until its assignment stabilizes, and +// asserts only the matching topics are present. A binding regression that +// passed the subscription string through unmodified (so `^` was lost) or +// that crossed up topic ownership would fail the topic-set comparison. +#[tokio::test(flavor = "multi_thread")] +async fn test_consumer_regex_subscription_matches_only_prefixed() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let prefix = rand_test_topic("regex_match"); + let match1 = format!("{}_a", prefix); + let match2 = format!("{}_b", prefix); + let nonmatch = format!("other_{}", prefix); + + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + for topic in [&match1, &match2, &nonmatch] { + admin_client + .create_topics( + &admin::new_topic_vec(topic, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + } + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("could not create future producer"); + for topic in [&match1, &match2, &nonmatch] { + producer + .send( + FutureRecord::to(topic.as_str()) + .partition(0) + .key("k") + .payload("p"), + Duration::from_secs(10), + ) + .await + .unwrap_or_else(|(e, _)| panic!("delivery failed: {}", e)); + } + + let consumer = utils::consumer::stream_consumer::create_stream_consumer( + &kafka_context.bootstrap_servers, + Some(&rand_test_group()), + ) + .await + .expect("could not create stream consumer"); + let pattern = format!("^{}.*", prefix); + consumer.subscribe(&[pattern.as_str()]).unwrap(); + + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let expected: std::collections::HashSet = + [match1.clone(), match2.clone()].into_iter().collect(); + let mut observed: std::collections::HashSet = std::collections::HashSet::new(); + while std::time::Instant::now() < deadline { + match time::timeout(Duration::from_secs(2), consumer.stream().next()).await { + Ok(Some(Ok(m))) => { + observed.insert(m.topic().to_string()); + } + Ok(Some(Err(e))) => panic!("stream error: {:?}", e), + Ok(None) => panic!("stream ended"), + Err(_) => {} + } + let assignment = consumer.assignment().unwrap(); + let assigned: std::collections::HashSet = assignment + .elements() + .iter() + .map(|e| e.topic().to_string()) + .collect(); + if assigned == expected { + break; + } + } + let assignment = consumer.assignment().unwrap(); + let assigned: std::collections::HashSet = assignment + .elements() + .iter() + .map(|e| e.topic().to_string()) + .collect(); + assert_eq!( + assigned, expected, + "regex subscription should converge to the two matching topics", + ); + assert!( + !observed.contains(&nonmatch), + "non-matching topic {} should not be delivered", + nonmatch + ); +} + // `test_produce_consume_with_timestamp` already calls `offsets_for_timestamp`, // but only with two distinct timestamp values. This test produces a strictly // monotonic sequence of distinct timestamps and queries at a midpoint, then From 40cf81469e34e857de86492b6961c526e8490eb9 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:31:51 -0500 Subject: [PATCH 50/56] test: assert rebalance_protocol() reports Cooperative for cooperative-sticky Adds `test_consumer_cooperative_sticky_rebalance_protocol` in tests/stream_consumers.rs. The existing `test_produce_consume_base_incremental_assign_and_unassign` exercises the incremental_assign/incremental_unassign API on a manually-assigned consumer (no group join, so rebalance_protocol stays None). This test joins a group with `partition.assignment.strategy=cooperative-sticky`, drives the consumer until both partitions are assigned, and asserts that `rebalance_protocol()` returns `None` before the join and `Cooperative` after. A binding regression in the `rebalance_protocol` accessor (or that dropped the cooperative-sticky configuration) would surface here. --- tests/stream_consumers.rs | 82 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/tests/stream_consumers.rs b/tests/stream_consumers.rs index d17cfa5f6..bc6047200 100644 --- a/tests/stream_consumers.rs +++ b/tests/stream_consumers.rs @@ -11,7 +11,7 @@ use rdkafka_sys::RDKafkaErrorCode; use tokio::time::{self, Duration}; use rdkafka::admin::AdminOptions; -use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer}; +use rdkafka::consumer::{CommitMode, Consumer, RebalanceProtocol, StreamConsumer}; use rdkafka::error::KafkaError; use rdkafka::message::{Header, Headers, OwnedHeaders}; use rdkafka::producer::FutureRecord; @@ -937,6 +937,86 @@ async fn test_consume_partition_order() { } } +// `test_produce_consume_base_incremental_assign_and_unassign` exercises the +// `incremental_assign`/`incremental_unassign` API on a manually-assigned +// consumer (no group join, so `rebalance_protocol` stays `None`). This test +// joins a group with `partition.assignment.strategy=cooperative-sticky`, drives +// the consumer until the initial assignment lands, and asserts that +// `rebalance_protocol()` reports `Cooperative`. A binding regression in the +// `rebalance_protocol` accessor (or that ignored the cooperative-sticky +// configuration) would fail this check. +#[tokio::test(flavor = "multi_thread")] +async fn test_consumer_cooperative_sticky_rebalance_protocol() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_consumer_cooperative_sticky"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(2)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let producer = producer::future_producer::create_producer(&kafka_context.bootstrap_servers) + .await + .expect("could not create future producer"); + for partition in 0..2 { + producer + .send( + FutureRecord::to(&topic_name) + .partition(partition) + .key("k") + .payload("p"), + Duration::from_secs(10), + ) + .await + .unwrap_or_else(|(e, _)| panic!("delivery failed: {}", e)); + } + + let consumer = utils::consumer::stream_consumer::create_stream_consumer_with_options( + &kafka_context.bootstrap_servers, + &rand_test_group(), + &[("partition.assignment.strategy", "cooperative-sticky")], + ) + .await + .expect("could not create stream consumer"); + consumer.subscribe(&[topic_name.as_str()]).unwrap(); + + assert!( + matches!(consumer.rebalance_protocol(), RebalanceProtocol::None), + "rebalance_protocol should be None before the first group join", + ); + + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while std::time::Instant::now() < deadline { + let _ = time::timeout(Duration::from_secs(2), consumer.stream().next()).await; + if consumer.assignment().unwrap().count() == 2 { + break; + } + } + let assignment = consumer.assignment().unwrap(); + assert_eq!( + assignment.count(), + 2, + "consumer should have been assigned both partitions, got {:?}", + assignment + ); + assert!( + matches!( + consumer.rebalance_protocol(), + RebalanceProtocol::Cooperative + ), + "rebalance_protocol should report Cooperative after a cooperative-sticky join", + ); +} + // librdkafka treats subscription strings beginning with `^` as a regex // pattern and resolves them against the broker's topic metadata on every // metadata refresh. This test creates two topics that match a unique From e54ac608b597adc63dd1bc8a2c0457b7b63195e4 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:33:01 -0500 Subject: [PATCH 51/56] test: cover alter_configs on a topic with DynamicTopic source readback Adds `test_alter_topic_configs_retention_ms_dynamic_topic` in tests/admin.rs. `test_configs` already covers broker-scoped alter_configs; this test covers the topic-scoped path by creating a fresh topic, altering `retention.ms` with `AlterConfig::new(ResourceSpecifier::Topic(...))`, then issuing `describe_configs` and asserting the entry value updated to the new number and that its `source` switched to `ConfigSource::DynamicTopic`. A binding regression that misrouted a topic-scoped AlterConfigs request to a broker handler, or that misclassified the readback source, would fail one of those assertions. --- tests/admin.rs | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/admin.rs b/tests/admin.rs index 932a33a98..21f9ee5bb 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -666,3 +666,72 @@ async fn test_event_errors() { Err(KafkaError::AdminOp(RDKafkaErrorCode::OperationTimedOut)) ); } + +// `test_configs` covers broker-scoped alter_configs; this test covers the +// topic-scoped path. It creates a fresh topic, alters `retention.ms` via +// `alter_configs` on a `ResourceSpecifier::Topic`, then issues +// `describe_configs` and asserts the entry value updated to the new number +// and its `source` switched to `ConfigSource::DynamicTopic`. A binding +// regression that misrouted the topic-scoped AlterConfigs request to a +// broker handler, or that misclassified the readback source, would fail +// here. +#[tokio::test] +async fn test_alter_topic_configs_retention_ms_dynamic_topic() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); + + let topic_name = rand_test_topic("test_alter_topic_retention"); + let create_results = admin_client + .create_topics(&utils::admin::new_topic_vec(&topic_name, Some(1)), &opts) + .await + .expect("could not create topic"); + assert_eq!(create_results, vec![Ok(topic_name.clone())]); + + let resource = ResourceSpecifier::Topic(&topic_name); + let new_value = "604800000"; + let config = AlterConfig::new(resource).set("retention.ms", new_value); + let alter_results = admin_client + .alter_configs(&[config], &opts) + .await + .expect("alter configs failed"); + assert_eq!( + alter_results, + vec![Ok(OwnedResourceSpecifier::Topic(topic_name.clone()))] + ); + + let mut tries = 0; + loop { + let describe_results = admin_client + .describe_configs(&[resource], &opts) + .await + .expect("describe configs failed"); + let cfg = describe_results[0] + .as_ref() + .expect("describe configs returned an error"); + let entry = cfg.get("retention.ms").expect("retention.ms entry missing"); + let expected = ConfigEntry { + name: "retention.ms".into(), + value: Some(new_value.into()), + source: ConfigSource::DynamicTopic, + is_read_only: false, + is_default: false, + is_sensitive: false, + }; + if entry == &expected { + break; + } else if tries >= 5 { + panic!("retention.ms did not converge: got {:?}", entry); + } else { + tries += 1; + tokio::time::sleep(Duration::from_secs(1)).await; + } + } +} From 13f85a3597e86dccb1083ffd34e06a0b0dfd4ae8 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:33:49 -0500 Subject: [PATCH 52/56] test: assert InvalidReplicationFactor on Fixed(3) against single broker Adds `test_create_topics_fixed_replication_too_high` in tests/admin.rs. `test_incorrect_replication_factors_are_ignored_when_creating_topics` already exercises the client-side validation path for a `TopicReplication::Variable` mismatch. This test covers the broker-side path: it asks for `TopicReplication::Fixed(3)` against the single-broker container, expects the broker to reject the request, and pins the surfaced error to `RDKafkaErrorCode::InvalidReplicationFactor`. A binding regression that swallowed the per-topic error or remapped the code would fail one of the assertions. --- tests/admin.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/admin.rs b/tests/admin.rs index 21f9ee5bb..c01d4722b 100644 --- a/tests/admin.rs +++ b/tests/admin.rs @@ -667,6 +667,50 @@ async fn test_event_errors() { ); } +// `test_incorrect_replication_factors_are_ignored_when_creating_topics` +// exercises the client-side validation for a `TopicReplication::Variable` +// mismatch. This test covers the broker-side path: it asks for +// `TopicReplication::Fixed(3)` against the single-broker container, expects +// the broker to reject the request, and pins the surfaced error to +// `RDKafkaErrorCode::InvalidReplicationFactor`. A binding regression that +// swallowed the per-topic error (returning `Ok` from `create_topics`) or +// remapped the code would fail one of those assertions. +#[tokio::test] +async fn test_create_topics_fixed_replication_too_high() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(30))); + + let topic_name = rand_test_topic("test_create_topics_fixed_replication_too_high"); + let topic = NewTopic::new(&topic_name, 1, TopicReplication::Fixed(3)); + let results = admin_client + .create_topics(&[topic], &opts) + .await + .expect("create_topics request itself should succeed"); + assert_eq!(results.len(), 1); + match &results[0] { + Err((name, code)) => { + assert_eq!(name, &topic_name); + assert_eq!( + *code, + RDKafkaErrorCode::InvalidReplicationFactor, + "expected InvalidReplicationFactor, got {:?}", + code + ); + } + Ok(_) => panic!( + "create_topics unexpectedly succeeded for replication-factor=3 on a single broker" + ), + } +} + // `test_configs` covers broker-scoped alter_configs; this test covers the // topic-scoped path. It creates a fresh topic, alters `retention.ms` via // `alter_configs` on a `ResourceSpecifier::Topic`, then issues From a78973dd51673501174f8bf7fb70d2e4afbc6f75 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:35:47 -0500 Subject: [PATCH 53/56] test: assert delete_groups returns NonEmptyGroup then succeeds after drop Adds `test_delete_non_empty_consumer_group` in tests/consumer_groups.rs. The test creates a topic and a `BaseConsumer` in a random group, subscribes the consumer and polls it until it has actually joined the group (assignment is non-empty), calls `delete_groups`, and asserts the per-group result is `Err((, NonEmptyGroup))`. It then drops the consumer (LeaveGroup) and retries `delete_groups` until the broker reports success or the 30s deadline elapses. A binding regression that misclassified the per-group error, or that failed to retry once the member had left, would surface here. --- tests/consumer_groups.rs | 101 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/tests/consumer_groups.rs b/tests/consumer_groups.rs index 603a688fa..e41325665 100644 --- a/tests/consumer_groups.rs +++ b/tests/consumer_groups.rs @@ -1,8 +1,11 @@ +use std::time::{Duration, Instant}; + use crate::utils::consumer; use crate::utils::containers::KafkaContext; use crate::utils::logging::init_test_logger; use crate::utils::rand::{rand_test_group, rand_test_topic}; use rdkafka::admin::{AdminOptions, GroupResult, NewTopic, TopicReplication}; +use rdkafka::consumer::Consumer; use rdkafka_sys::RDKafkaErrorCode; mod utils; @@ -95,6 +98,104 @@ pub async fn test_delete_unknown_group() { ); } +// `delete_groups` cannot remove a group while it still has an active member. +// This test subscribes a consumer to a topic, drives it until it has actually +// joined the group, calls `delete_groups`, and asserts the per-group result +// is `NonEmptyGroup`. It then drops the consumer (which sends LeaveGroup), +// retries `delete_groups`, and asserts the second call succeeds. A binding +// regression that misclassified the per-group error or that lost the +// active-membership signal in the second-call retry would fail one of those +// assertions. +#[tokio::test] +pub async fn test_delete_non_empty_consumer_group() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + + let admin_client = utils::admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + + let group_name = rand_test_group(); + let topic_name = rand_test_topic("test_delete_non_empty_group"); + + admin_client + .create_topics( + &[NewTopic { + name: &topic_name, + num_partitions: 1, + replication: TopicReplication::Fixed(1), + config: vec![], + }], + &AdminOptions::default(), + ) + .await + .expect("topic creation failed"); + + let consumer_client = utils::consumer::create_subscribed_base_consumer( + &kafka_context.bootstrap_servers, + Some(&group_name), + &topic_name, + ) + .await + .expect("could not create subscribed consumer"); + + let deadline = Instant::now() + Duration::from_secs(20); + loop { + consumer_client.poll(Duration::from_millis(200)); + if consumer_client.assignment().unwrap().count() > 0 { + break; + } + if Instant::now() > deadline { + panic!("consumer never joined the group"); + } + } + + let res = admin_client + .delete_groups(&[&group_name], &AdminOptions::default()) + .await + .expect("delete_groups call should not itself fail"); + let first: &GroupResult = res.first().expect("expected one result"); + let (returned_name, code) = first + .as_ref() + .expect_err("delete_groups on a non-empty group should be an error"); + assert_eq!(returned_name, &group_name); + assert_eq!( + *code, + RDKafkaErrorCode::NonEmptyGroup, + "expected NonEmptyGroup while the consumer is still active, got {:?}", + code + ); + + drop(consumer_client); + + let deadline = Instant::now() + Duration::from_secs(30); + let last_err: RDKafkaErrorCode = loop { + let res = admin_client + .delete_groups(&[&group_name], &AdminOptions::default()) + .await + .expect("delete_groups call should not itself fail"); + match res.first().expect("expected one result") { + Ok(name) => { + assert_eq!(name, &group_name); + return; + } + Err((_, code)) => { + if Instant::now() > deadline { + break *code; + } + } + } + tokio::time::sleep(Duration::from_millis(500)).await; + }; + panic!( + "delete_groups never converged to success after consumer drop (last error: {:?})", + last_err + ); +} + /// Verify that deleting a valid and invalid group results in a mixed result /// set. #[tokio::test] From baf27e448420d664bb9cfafd043cc3f6c64426a7 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:36:52 -0500 Subject: [PATCH 54/56] test: assert isolation.level filters interleaved aborted vs committed payloads Adds `test_transaction_isolation_level_filters_aborted` in tests/transactions.rs. `test_transaction_abort` and `test_transaction_commit` each cover a single transaction in isolation, but they never interleave committed and aborted records on the same topic-partition. This test runs two transactional producers (different `transactional.id`) into the same topic: one commits 7 payloads, the other aborts 5. A `read_committed` consumer must see exactly the 7 committed payloads in order, and a `read_uncommitted` consumer must see all 12 payloads (committed followed by aborted). A binding regression on the `isolation.level` plumbing, or a librdkafka filter break that included or dropped aborted records, would surface as the wrong payload set. --- tests/transactions.rs | 111 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/tests/transactions.rs b/tests/transactions.rs index e42c4f64a..fe14c8d2e 100644 --- a/tests/transactions.rs +++ b/tests/transactions.rs @@ -311,6 +311,117 @@ async fn test_transaction_commit() -> Result<(), Box> { Ok(()) } +// `test_transaction_abort` and `test_transaction_commit` each cover a single +// producer running a single transaction, but they never interleave committed +// and aborted records on the same topic-partition. This test mixes the two: +// one transactional producer commits 7 records, a second transactional +// producer (different `transactional.id`) aborts 5 records on the same topic. +// A `read_committed` consumer must observe exactly the 7 committed payloads +// (the broker writes a control marker that the consumer-side filter must +// honour); a `read_uncommitted` consumer must observe all 12 payloads. A +// binding regression on the `isolation.level` plumbing, or a librdkafka +// filter break, would show up as the wrong total or the wrong payload set. +#[tokio::test] +async fn test_transaction_isolation_level_filters_aborted() -> Result<(), Box> { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic = rand_test_topic("test_txn_isolation_filter"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let committed_producer = create_producer(&kafka_context)?; + committed_producer.init_transactions(Timeout::Never)?; + committed_producer.begin_transaction()?; + let committed_payloads: Vec = (0..7).map(|i| format!("committed-{}", i)).collect(); + for payload in &committed_payloads { + committed_producer + .send( + BaseRecord::to(&topic) + .payload(payload.as_str()) + .key("k") + .partition(0), + ) + .map_err(|(e, _)| e)?; + } + committed_producer.flush(Duration::from_secs(20))?; + committed_producer.commit_transaction(Duration::from_secs(20))?; + + let aborted_producer = create_producer(&kafka_context)?; + aborted_producer.init_transactions(Timeout::Never)?; + aborted_producer.begin_transaction()?; + let aborted_payloads: Vec = (0..5).map(|i| format!("aborted-{}", i)).collect(); + for payload in &aborted_payloads { + aborted_producer + .send( + BaseRecord::to(&topic) + .payload(payload.as_str()) + .key("k") + .partition(0), + ) + .map_err(|(e, _)| e)?; + } + aborted_producer.flush(Duration::from_secs(20))?; + aborted_producer.abort_transaction(Duration::from_secs(20))?; + + let collect_payloads = |iso: IsolationLevel| { + let iso_str = match iso { + IsolationLevel::ReadCommitted => "read_committed", + IsolationLevel::ReadUncommitted => "read_uncommitted", + }; + let kafka_context = kafka_context.clone(); + let topic = topic.clone(); + async move { + let consumer = create_consumer( + &kafka_context, + Some(&[ + ("isolation.level", iso_str), + ("enable.partition.eof", "true"), + ]), + ) + .await?; + let mut tpl = TopicPartitionList::new(); + tpl.add_partition(&topic, 0); + consumer.assign(&tpl)?; + let mut payloads = Vec::new(); + for message in consumer.iter() { + match message { + Ok(m) => payloads.push(m.payload_view::().unwrap().unwrap().to_string()), + Err(KafkaError::PartitionEOF(_)) => break, + Err(e) => return Err(e), + } + } + Ok::<_, KafkaError>(payloads) + } + }; + + let committed_view = collect_payloads(IsolationLevel::ReadCommitted).await?; + assert_eq!( + committed_view, committed_payloads, + "read_committed should see only the committed payloads in order" + ); + + let uncommitted_view = collect_payloads(IsolationLevel::ReadUncommitted).await?; + let mut expected_uncommitted = committed_payloads.clone(); + expected_uncommitted.extend(aborted_payloads.iter().cloned()); + assert_eq!( + uncommitted_view, expected_uncommitted, + "read_uncommitted should see committed payloads followed by aborted payloads" + ); + + Ok(()) +} + // `test_transaction_commit` already calls `send_offsets_to_transaction`, but // it produces a fixed "A" payload regardless of what was consumed and only // looks at offsets at the end. This test does a full consume-transform-produce From 8178ba04267e75cbd2876eaa991d7327f99d99c1 Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:38:21 -0500 Subject: [PATCH 55/56] test: assert second init_transactions fences the first producer's commit Adds `test_transaction_producer_fenced_by_epoch` in tests/transactions.rs. The test creates two transactional producers that share the same `transactional.id`, drives the first through `init_transactions`/`begin_transaction`/`send`, then runs `init_transactions` on the second producer to bump the epoch on the broker. The first producer's subsequent `commit_transaction` is then expected to fail with a `KafkaError::Transaction` whose code is one of the librdkafka fencing codes (`Fenced`, `InvalidProducerEpoch`, or `ProducerFenced`). A binding regression that pretended the fenced commit succeeded, or that mapped the fencing code into a different error variant, would surface here. --- tests/transactions.rs | 85 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/tests/transactions.rs b/tests/transactions.rs index fe14c8d2e..c1856f0ee 100644 --- a/tests/transactions.rs +++ b/tests/transactions.rs @@ -9,7 +9,7 @@ use rdkafka::admin::AdminOptions; use rdkafka::config::ClientConfig; use rdkafka::config::RDKafkaLogLevel; use rdkafka::consumer::{BaseConsumer, CommitMode, Consumer}; -use rdkafka::error::KafkaError; +use rdkafka::error::{KafkaError, RDKafkaErrorCode}; use rdkafka::message::Message; use rdkafka::producer::{BaseProducer, BaseRecord, Producer}; use rdkafka::topic_partition_list::{Offset, TopicPartitionList}; @@ -311,6 +311,89 @@ async fn test_transaction_commit() -> Result<(), Box> { Ok(()) } +fn create_producer_with_txn_id( + kafka_context: &KafkaContext, + transactional_id: &str, +) -> Result { + let mut config = ClientConfig::new(); + config + .set("bootstrap.servers", &kafka_context.bootstrap_servers) + .set("message.timeout.ms", "5000") + .set("enable.idempotence", "true") + .set("transactional.id", transactional_id); + config.create() +} + +// When two producers share a `transactional.id`, the broker fences the older +// epoch on the second producer's `init_transactions`. The fenced producer's +// next transactional API call must surface that fact rather than silently +// committing. This test sets up two producers with the same transactional +// id, drives the second through `init_transactions` (which fences the +// first), and asserts the first's `commit_transaction` returns a +// `KafkaError::Transaction` whose underlying code is one of the librdkafka +// fencing codes. A binding regression that hid the fencing error or +// pretended the commit succeeded would surface here. +#[tokio::test] +async fn test_transaction_producer_fenced_by_epoch() -> Result<(), Box> { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic = rand_test_topic("test_txn_fencing"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let txn_id = rand_test_transactional_id(); + + let first = create_producer_with_txn_id(&kafka_context, &txn_id)?; + first.init_transactions(Timeout::Never)?; + first.begin_transaction()?; + first + .send( + BaseRecord::to(&topic) + .payload("first-pre-fence") + .key("k") + .partition(0), + ) + .map_err(|(e, _)| e)?; + first.flush(Duration::from_secs(20))?; + + let second = create_producer_with_txn_id(&kafka_context, &txn_id)?; + second.init_transactions(Timeout::Never)?; + drop(second); + + let result = first.commit_transaction(Duration::from_secs(20)); + match result { + Ok(()) => panic!("commit_transaction unexpectedly succeeded after the producer was fenced"), + Err(KafkaError::Transaction(rd_err)) => { + let code = rd_err.code(); + assert!( + matches!( + code, + RDKafkaErrorCode::Fenced + | RDKafkaErrorCode::InvalidProducerEpoch + | RDKafkaErrorCode::ProducerFenced + ), + "expected a producer-fencing error code, got {:?} ({})", + code, + rd_err.string(), + ); + } + Err(other) => panic!("unexpected error variant: {:?}", other), + } + + Ok(()) +} + // `test_transaction_abort` and `test_transaction_commit` each cover a single // producer running a single transaction, but they never interleave committed // and aborted records on the same topic-partition. This test mixes the two: From ddd9961356ec34ee3207c96afbe608b00dbc0a7a Mon Sep 17 00:00:00 2001 From: Johnathan W Date: Fri, 22 May 2026 10:39:14 -0500 Subject: [PATCH 56/56] test: assert ClientContext::stats fires on statistics.interval.ms cadence Adds `test_base_producer_statistics_callback_invoked` in tests/base_producer.rs. The test wires the existing `CollectingContext`, configures `statistics.interval.ms=100`, runs a send loop for ~400ms, and then asserts at least two stats callbacks landed and that each delivered `Statistics` carries a non-empty `name` and a `client_type` of `"producer"`. A binding regression that swallowed the stats callback, or that deserialised the JSON into a partial/default struct, would fail one of those assertions. --- tests/base_producer.rs | 69 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/base_producer.rs b/tests/base_producer.rs index d637ccb49..515b6766c 100644 --- a/tests/base_producer.rs +++ b/tests/base_producer.rs @@ -623,6 +623,75 @@ async fn test_custom_partitioner_base_producer() { } } +// `statistics.interval.ms=100` directs librdkafka to invoke +// `ClientContext::stats` every 100ms. This test runs a send loop for ~400ms +// against a real broker, asserts the callback fired at least twice, and (via +// the `Statistics` struct that the binding already deserialises into) that +// each delivered struct carries non-empty top-level metadata. A binding +// regression that swallowed the stats callback or deserialised the JSON into +// a partial / default struct would fail one of those assertions. +#[tokio::test(flavor = "multi_thread")] +async fn test_base_producer_statistics_callback_invoked() { + init_test_logger(); + + let kafka_context = KafkaContext::shared() + .await + .expect("could not create kafka context"); + let topic_name = rand_test_topic("test_base_producer_statistics_callback_invoked"); + let admin_client = admin::create_admin_client(&kafka_context.bootstrap_servers) + .await + .expect("could not create admin client"); + admin_client + .create_topics( + &admin::new_topic_vec(&topic_name, Some(1)), + &AdminOptions::default(), + ) + .await + .expect("could not create topic"); + + let context = CollectingContext::new(); + let producer = base_producer_utils::create_base_producer_with_context( + &kafka_context.bootstrap_servers, + context.clone(), + &[("statistics.interval.ms", "100")], + ) + .expect("failed to create base producer"); + + let start = std::time::Instant::now(); + while start.elapsed() < Duration::from_millis(400) { + producer + .send( + BaseRecord::with_opaque_to(&topic_name, 0usize) + .payload("p") + .key("k"), + ) + .expect("send failed"); + producer.poll(Duration::from_millis(50)); + } + producer + .flush(Duration::from_secs(10)) + .expect("flush failed"); + + let stats = context.stats.lock().unwrap(); + assert!( + stats.len() >= 2, + "expected at least two stats callbacks over 400ms with interval 100ms, got {}", + stats.len() + ); + for snapshot in stats.iter() { + assert!( + !snapshot.name.is_empty(), + "Statistics.name should be populated: {:?}", + snapshot + ); + assert_eq!( + snapshot.client_type, "producer", + "Statistics.client_type should report producer: {:?}", + snapshot.client_type + ); + } +} + // Asserts the synchronous contract of `Producer::flush`: when `flush(timeout)` // returns successfully, every previously-queued record has reached its // delivery callback and the in-flight counter is zero. A binding regression