diff --git a/build.gradle.kts b/build.gradle.kts index 5745a753b3ab..9cbf5d760a0a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -218,6 +218,9 @@ tasks.rat { // Ignore poetry autogenerated files. "**/poetry.lock", + // Ignore Cargo lock files. + "sdks/rust/Cargo.lock", + // DuetAI training prompts "learning/prompts/**/*.md", diff --git a/local-env-setup.sh b/local-env-setup.sh index 0c28b4e65f82..0cf0ff1827d3 100755 --- a/local-env-setup.sh +++ b/local-env-setup.sh @@ -68,6 +68,13 @@ if [ "$kernelname" = "Linux" ]; then exit fi + type -P rustc > /dev/null 2>&1 + rustExists=$? + if [ $rustExists -ne 0 ]; then + echo "Installing rust via rustup..." + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + fi + # Running on Mac elif [ "$kernelname" = "Darwin" ]; then # Check for Homebrew, install if we don't have it @@ -144,5 +151,23 @@ elif [ "$kernelname" = "Darwin" ]; then exit fi + type -P rustc > /dev/null 2>&1 + rustExists=$? + if [ $rustExists -eq 0 ]; then + echo "rust already installed. Skipping" + else + echo "Installing rust" + brew install rust + fi + + type -P protoc > /dev/null 2>&1 + protocExists=$? + if [ $protocExists -eq 0 ]; then + echo "protobuf already installed. Skipping" + else + echo "Installing protobuf" + brew install protobuf + fi + else echo "Unrecognized Kernel Name: $kernelname" fi diff --git a/release/src/main/scripts/set_version.sh b/release/src/main/scripts/set_version.sh index 138275f20a32..681b83261556 100755 --- a/release/src/main/scripts/set_version.sh +++ b/release/src/main/scripts/set_version.sh @@ -92,11 +92,13 @@ if [[ -z "$IS_SNAPSHOT_VERSION" ]] ; then sed -i -e "s/SdkVersion = .*/SdkVersion = \"$TARGET_VERSION\"/" sdks/go/pkg/beam/core/core.go sed -i -e "s/\"version\": .*/\"version\": \"$TARGET_VERSION\",/" sdks/typescript/package.json sed -i -e "s/DEFAULT_BEAM_VERSION=\".*\"/DEFAULT_BEAM_VERSION=\"$TARGET_VERSION\"/" scripts/beam-sql.sh + sdks/rust/scripts/set_version.sh "$TARGET_VERSION" else # For snapshot version: # Java/gradle appends -SNAPSHOT # In the Gradle plugin, the -SNAPSHOT is dynamic so we don't add it here # Python appends .dev + # Rust appends -SNAPSHOT (Cargo requires semver) # The Dataflow container remains unchanged as in this case it is beam-master- form sed -i -e "s/version=.*/version=$TARGET_VERSION-SNAPSHOT/" gradle.properties sed -i -e "s/project.version = .*/project.version = '$TARGET_VERSION'/" buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -105,6 +107,7 @@ else sed -i -e "s/SdkVersion = .*/SdkVersion = \"${TARGET_VERSION}.dev\"/" sdks/go/pkg/beam/core/core.go sed -i -e "s/\"version\": .*/\"version\": \"$TARGET_VERSION-SNAPSHOT\",/" sdks/typescript/package.json sed -i -e "s/DEFAULT_BEAM_VERSION=\".*\"/DEFAULT_BEAM_VERSION=\"$TARGET_VERSION\"/" scripts/beam-sql.sh + sdks/rust/scripts/set_version.sh "$TARGET_VERSION-SNAPSHOT" fi if [[ "$GIT_ADD" == yes ]] ; then @@ -115,4 +118,5 @@ if [[ "$GIT_ADD" == yes ]] ; then git add runners/google-cloud-dataflow-java/build.gradle git add sdks/typescript/package.json git add scripts/beam-sql.sh + git add 'sdks/rust/*Cargo.toml' sdks/rust/Cargo.lock fi diff --git a/sdks/rust/Cargo.lock b/sdks/rust/Cargo.lock new file mode 100644 index 000000000000..793a0756b133 --- /dev/null +++ b/sdks/rust/Cargo.lock @@ -0,0 +1,937 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "apache-beam-model" +version = "2.78.0-SNAPSHOT" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-prost", + "tonic-prost-build", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab1ad36992cead65f02aa399a373a42730922f1525d988172634fdefdecb8a60" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/sdks/rust/Cargo.toml b/sdks/rust/Cargo.toml new file mode 100644 index 000000000000..978bab63ddb8 --- /dev/null +++ b/sdks/rust/Cargo.toml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +resolver = "3" +members = [ + "beam/model", +] + +[workspace.package] +version = "2.78.0-SNAPSHOT" +edition = "2024" +rust-version = "1.98" +license = "Apache-2.0" +repository = "https://github.com/apache/beam" +homepage = "https://beam.apache.org" + +# Unused-code lints that rustc leaves allow-by-default, so `-D warnings` does not reach +# them. Declared here rather than on the Gradle command line so the IDE and a bare +# `cargo clippy` enforce the same set. Each member opts in with `[lints] workspace = true`. +[workspace.lints.rust] +unused_qualifications = "warn" +unused_lifetimes = "warn" +unused_import_braces = "warn" +unused_macro_rules = "warn" +single_use_lifetimes = "warn" + +[workspace.lints.clippy] +redundant_clone = "warn" +# Suppressions must say why, and should be `#[expect]` so stale ones are reported. +allow_attributes_without_reason = "warn" +# Library code reports errors instead of panicking; see clippy.toml for tests. +unwrap_used = "warn" +# Leftover debugging. +dbg_macro = "warn" +todo = "warn" +# Style. +uninlined_format_args = "warn" +manual_let_else = "warn" +semicolon_if_nothing_returned = "warn" + +[workspace.dependencies] +prost = "0.14" +prost-types = "0.14" +tonic = "0.14" +tonic-prost = "0.14" +tonic-prost-build = "0.14" + +# Keep file:line in panic backtraces without full DWARF debug info. +[profile.dev] +debug = "line-tables-only" + +# Omit debug info for third-party dependencies to speed up linking. +[profile.dev.package."*"] +debug = false + +# The test profile inherits from dev, so it picks up both settings above. diff --git a/sdks/rust/README.md b/sdks/rust/README.md new file mode 100644 index 000000000000..433dcd919a71 --- /dev/null +++ b/sdks/rust/README.md @@ -0,0 +1,49 @@ + + +# Rust SDK + +A library for writing [Apache Beam](https://beam.apache.org/) pipelines in Rust. + +The SDK is under active development. [DESIGN.md](DESIGN.md) describes its design. + +## Crate Layout + +The SDK is a Cargo workspace with its root at `sdks/rust`: + +| Crate | Path | Contents | +| :--- | :--- | :--- | +| `apache-beam-model` | `beam/model/` | Protobuf and gRPC bindings generated from the standard Beam model protos. | + +## Building and Testing + +Run builds through Gradle, like the rest of the Beam repository: + +```bash +./gradlew :sdks:rust:build # Compile all crates +./gradlew :sdks:rust:test # Run all tests (one crate: -Ppkg=) +./gradlew :sdks:rust:check # rustfmt --check + clippy -D warnings +./gradlew :sdks:rust:fmt # Apply formatting +``` + +For the full task reference, see [docs/development.md](docs/development.md). + +## Contributing + +See the [Beam Contribution Guide](https://beam.apache.org/contribute/). Each new file must have the standard Apache 2.0 license header. Before you open a PR, run `./gradlew :sdks:rust:check` to verify compliance. diff --git a/sdks/rust/beam/model/Cargo.toml b/sdks/rust/beam/model/Cargo.toml new file mode 100644 index 000000000000..f97efdc3e7e4 --- /dev/null +++ b/sdks/rust/beam/model/Cargo.toml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "apache-beam-model" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Portability model definitions and gRPC bindings for Apache Beam in Rust" + +[lib] +name = "model" +doctest = false + +[dependencies] +prost = { workspace = true } +prost-types = { workspace = true } +tonic = { workspace = true } +tonic-prost = { workspace = true } + +[build-dependencies] +tonic-prost-build = { workspace = true } + +# No [lints] workspace opt-in: this crate is entirely tonic/prost generated code. + +# These are used by the generated code `tonic::include_proto!` pulls in from OUT_DIR, +# which cargo-machete does not scan. +[package.metadata.cargo-machete] +ignored = ["prost", "prost-types", "tonic-prost"] + diff --git a/sdks/rust/beam/model/build.rs b/sdks/rust/beam/model/build.rs new file mode 100644 index 000000000000..c02b4a749740 --- /dev/null +++ b/sdks/rust/beam/model/build.rs @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use std::path::{Path, PathBuf}; + +fn main() -> Result<(), Box> { + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?); + let root_dir = manifest_dir.join("../../../.."); + + let pipeline_proto_dir = root_dir.join("model/pipeline/src/main/proto"); + let fn_execution_proto_dir = root_dir.join("model/fn-execution/src/main/proto"); + let job_management_proto_dir = root_dir.join("model/job-management/src/main/proto"); + + let proto_sets: &[(&Path, &[&str])] = &[ + ( + &pipeline_proto_dir, + &[ + "org/apache/beam/model/pipeline/v1/endpoints.proto", + "org/apache/beam/model/pipeline/v1/schema.proto", + "org/apache/beam/model/pipeline/v1/metrics.proto", + "org/apache/beam/model/pipeline/v1/standard_window_fns.proto", + "org/apache/beam/model/pipeline/v1/beam_runner_api.proto", + "org/apache/beam/model/pipeline/v1/external_transforms.proto", + ], + ), + ( + &job_management_proto_dir, + &[ + "org/apache/beam/model/job_management/v1/beam_artifact_api.proto", + "org/apache/beam/model/job_management/v1/beam_expansion_api.proto", + "org/apache/beam/model/job_management/v1/beam_job_api.proto", + ], + ), + ( + &fn_execution_proto_dir, + &[ + "org/apache/beam/model/fn_execution/v1/beam_provision_api.proto", + "org/apache/beam/model/fn_execution/v1/beam_fn_api.proto", + ], + ), + ]; + + let protos: Vec = proto_sets + .iter() + .flat_map(|(dir, files)| files.iter().map(move |file| dir.join(file))) + .collect(); + + let includes = [ + pipeline_proto_dir, + fn_execution_proto_dir, + job_management_proto_dir, + ]; + + for proto in &protos { + println!("cargo:rerun-if-changed={}", proto.display()); + } + + tonic_prost_build::configure() + .build_server(true) + .build_client(true) + .compile_protos(&protos, &includes)?; + + Ok(()) +} diff --git a/sdks/rust/beam/model/src/lib.rs b/sdks/rust/beam/model/src/lib.rs new file mode 100644 index 000000000000..be6579e880a9 --- /dev/null +++ b/sdks/rust/beam/model/src/lib.rs @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Generated Protobuf and gRPC definitions for Apache Beam. + +#![allow(clippy::all)] +#![allow(unknown_lints)] +#![allow(unused_imports)] +#![allow(non_snake_case)] + +pub mod org { + pub mod apache { + pub mod beam { + pub mod model { + pub mod pipeline { + pub mod v1 { + tonic::include_proto!("org.apache.beam.model.pipeline.v1"); + } + } + pub mod fn_execution { + pub mod v1 { + tonic::include_proto!("org.apache.beam.model.fn_execution.v1"); + } + } + pub mod job_management { + pub mod v1 { + tonic::include_proto!("org.apache.beam.model.job_management.v1"); + } + } + pub mod expansion { + pub mod v1 { + tonic::include_proto!("org.apache.beam.model.expansion.v1"); + } + } + } + } + } +} + +// Short top-level paths to the versioned proto modules. +pub mod pipeline { + pub use crate::org::apache::beam::model::pipeline::v1::*; +} + +pub mod fn_execution { + pub use crate::org::apache::beam::model::fn_execution::v1::*; +} + +pub mod job_management { + pub use crate::org::apache::beam::model::job_management::v1::*; +} + +pub mod expansion { + pub use crate::org::apache::beam::model::expansion::v1::*; +} diff --git a/sdks/rust/build.gradle b/sdks/rust/build.gradle new file mode 100644 index 000000000000..07063a7ed040 --- /dev/null +++ b/sdks/rust/build.gradle @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Gradle entry point for the Rust SDK. Each area of the build is a script in gradle/. +// Cargo does the real work; these tasks give maintainers the same commands as the rest +// of the repository (`./gradlew :sdks:rust:build`, `:check`, `:test`). + +plugins { id 'org.apache.beam.module' } + +description = "Apache Beam :: SDKs :: Rust" + +apply from: 'gradle/versions.gradle' +apply from: 'gradle/cargo.gradle' diff --git a/sdks/rust/clippy.toml b/sdks/rust/clippy.toml new file mode 100644 index 000000000000..baa1f4dbcad5 --- /dev/null +++ b/sdks/rust/clippy.toml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# `clippy::unwrap_used` targets library code. This exempts `#[test]` functions and +# `#[cfg(test)]` modules; integration-test helpers opt out with a file-level `#![expect]`. +allow-unwrap-in-tests = true diff --git a/sdks/rust/docs/development.md b/sdks/rust/docs/development.md new file mode 100644 index 000000000000..64260857dae1 --- /dev/null +++ b/sdks/rust/docs/development.md @@ -0,0 +1,92 @@ + + +# Developing the Rust SDK + +This page gives the build, test and lint tasks for developers of the +SDK itself. Pipeline authors do not need these tasks. See the [SDK README](../README.md). + +Run builds and tests through Gradle, like the rest of the Beam repository. The Gradle tasks call cargo, so a cargo command gives the same result: + +```bash +# Core Lifecycle Tasks +./gradlew :sdks:rust:build # Compile all crates in workspace +./gradlew :sdks:rust:check # Run formatting checks and clippy linter +./gradlew :sdks:rust:test # Run all tests (or -Ppkg=apache-beam-model) +./gradlew :sdks:rust:fmt # Apply code formatting (cargo fmt) +./gradlew :sdks:rust:clean # Remove the cargo target directory +``` + +## One way + +Every public API change follows these rules: + +- **No aliases or compatibility shims.** A rename removes the old name. The same + change updates every caller, example and doc. +- **One public path per item.** Pipeline items are in their module + (`transforms`, `transforms::sdf`, `windowing`, `coders`, ...). Runner and + harness plumbing (`BundleHandler`, `ElementSink`, `HandlerContext`, ...) is + only in `beam::internals`. The preludes are the only additional path. +- **Fluent methods mirror core transforms.** Each method is the snake_case name + of exactly one core transform (`.group_by_key` → `GroupByKey`). It adds no + semantics. Documented exception: `with_side_singleton` / `with_side_iter` / + `with_side_map`. Each of these is a `ParDo` with one side input. +- **Fluent covers common cases.** For advanced configuration, use + `.apply(Transform::new(..))`, which also chains. Do not add fluent variants + for advanced configuration. +- **Construction.** Build transforms with `new(name, required..)` and + `.with_*()` builders. Value types (e.g. `FixedWindows::of`) are exempt. +- **Errors.** User hooks (`DoFn`, `CombineFn`, state, timers) return `beam::Result`. +- **Prelude.** `beam::prelude` contains only the items that a typical pipeline author uses. + +## Comments and documentation + +Comments and rustdoc follow the writing rules of +[ASD-STE100 Simplified Technical English](https://www.asd-ste100.org/about_STE.html). +Beam, Rust and protocol terms are permitted technical names. + +- **Short sentences.** At most 20 words for an instruction and 25 words for a + description. One topic per sentence. +- **Active voice.** Write instructions in the imperative: "Call `finish` before + `drop`", "Do not share the sink". +- **One word, one meaning.** Use the same term for the same thing in all files. +- **Say what the code cannot.** Write a comment for a constraint, a contract or a + reason that is not clear from the code. Do not repeat what the code does. +- **No numbered steps.** If the order is important, say so in words or split the + code into functions. +- **No history.** Do not write "used to", "now", "previously", "no longer" or + "Regression:". Write the rule that is true for the current code. A test + comment states the behavior that the test checks. +- **This SDK only.** Do not compare with other SDKs ("like Java", "same as Go", + "the equivalent of"). Name another SDK only when the behavior depends on it: + a cross-language transform that the code calls, or a wire format, URN or + protocol rule that other SDKs must decode. Comparisons between SDKs go in + `DESIGN.md`. +- **Plain words.** Use the simplest word that is correct. Do not use: thus, hence, + therefore, obey, ensure, utilize, leverage, facilitate, via, prior to, + subsequent, terminate, approximately, sufficient, robust, seamless, + comprehensive, crucial, essential, simply, merely, basically, actually, + effectively, notably, furthermore, moreover, additionally, "in order to", + "note that", "stem from". Use so, follow, make sure, use, help, through, + before, then, stop, about, enough, also, to. +- **Short.** A comment is shorter than the code it explains. Fill lines to about + 90 columns: one short sentence does not need its own line. +- **Rustdoc.** Start with a one-line summary. Do not repeat the item name + ("Returns the key." on `fn key`). Add `# Errors`, `# Panics` or `# Safety` only + when the condition is not clear from the signature and the summary. diff --git a/sdks/rust/gradle/cargo.gradle b/sdks/rust/gradle/cargo.gradle new file mode 100644 index 000000000000..09618b18b09c --- /dev/null +++ b/sdks/rust/gradle/cargo.gradle @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Lifecycle tasks: build, check (rustfmt + clippy), test, fmt and clean. +// +// The tasks always run cargo. Cargo has its own incremental build, so a run with nothing to +// do is fast, and its result is always the real result of the command. + +ext.rustDir = file("${rootDir}/sdks/rust") + +// -Ppkg= narrows `test` to one crate; the default is the whole workspace. +def selectedPkg = (project.findProperty('pkg') ?: project.findProperty('package'))?.toString() +ext.cargoScope = selectedPkg ? ['-p', selectedPkg] : ['--workspace'] +// -PtestArgs="" passes test-binary arguments after `--`, e.g. -PtestArgs=--ignored. +def testArgs = project.findProperty('testArgs')?.toString()?.tokenize(' ') ?: [] + +// Registers an Exec task that runs `cargo ` in the workspace root. +ext.cargoTask = { String name, String taskGroup, String text, List args -> + tasks.register(name, Exec) { + group = taskGroup + description = text + workingDir rustDir + commandLine(['cargo'] + args) + } +} + +cargoTask('buildWorkspace', 'Build', 'Compile all crates in the Rust workspace', + ['build', '--workspace']) +cargoTask('fmtCheck', 'Verification', 'Verify formatting (cargo fmt --check)', + ['fmt', '--check']) +cargoTask('fmt', 'Verification', 'Format all Rust code (cargo fmt)', + ['fmt']) +cargoTask('clippy', 'Verification', 'Run clippy on all targets (-D warnings)', + ['clippy', '--workspace', '--all-targets', '--', '-D', 'warnings']) +cargoTask('test', 'Verification', + 'Run tests (the workspace, or one crate with -Ppkg=; -PtestArgs= after --)', + ['test'] + cargoScope + (testArgs ? ['--'] + testArgs : [])) +cargoTask('cleanSdk', 'Build', 'Remove the cargo target directory', + ['clean']) + +build.dependsOn 'buildWorkspace' +check.dependsOn 'fmtCheck', 'clippy' +clean.dependsOn 'cleanSdk' diff --git a/sdks/rust/gradle/versions.gradle b/sdks/rust/gradle/versions.gradle new file mode 100644 index 000000000000..a5f82bea7f92 --- /dev/null +++ b/sdks/rust/gradle/versions.gradle @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Beam versions, all derived from gradle.properties (bumped by +// release/src/main/scripts/set_version.sh), and passed to every command this project runs. +// +// beamVersion the Gradle project version, e.g. X.Y.Z-SNAPSHOT or X.Y.Z +// beamReleaseVersion the release it becomes, e.g. X.Y.Z; names published release artifacts +// beamSdkVersion the SDK version, e.g. X.Y.Z.dev; stamped into binaries and container tags +// +// A plain `cargo` build reads the same gradle.properties keys (see beam/core/build.rs), so +// both paths report the same version. -Pver and -PbeamReleaseVersion override them here. + +ext.beamVersion = (project.findProperty('ver') ?: project.version).toString() +ext.beamReleaseVersion = (project.findProperty('beamReleaseVersion') + ?: beamVersion.replace('-SNAPSHOT', '')).toString() +ext.beamSdkVersion = project.property('sdk_version').toString() + +tasks.withType(Exec).configureEach { + environment 'BEAM_SDK_VERSION', beamSdkVersion + environment 'BEAM_RELEASE_VERSION', beamReleaseVersion + environment 'BEAM_VERSION', beamVersion +} diff --git a/sdks/rust/rustfmt.toml b/sdks/rust/rustfmt.toml new file mode 100644 index 000000000000..f89006ae7849 --- /dev/null +++ b/sdks/rust/rustfmt.toml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +edition = "2024" +max_width = 100 +use_small_heuristics = "Default" +newline_style = "Unix" +reorder_imports = true +reorder_modules = true diff --git a/sdks/rust/scripts/set_version.sh b/sdks/rust/scripts/set_version.sh new file mode 100755 index 000000000000..0e6f2b95b25d --- /dev/null +++ b/sdks/rust/scripts/set_version.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Sets the version of every crate in the Rust SDK workspace. +# +# Usage: set_version.sh +# +# Called by release/src/main/scripts/set_version.sh with a release version +# (X.Y.Z) or a snapshot version (X.Y.Z-SNAPSHOT). Cargo requires semver, so +# snapshots use a -SNAPSHOT pre-release suffix. +# +# Updates: +# - [workspace.package] version in Cargo.toml, +# - the version on every internal path dependency in any Cargo.toml, +# - workspace member entries in Cargo.lock ([[package]] blocks without a +# `source`), so `--locked` builds work without running cargo. + +set -euo pipefail + +if [[ $# -ne 1 || -z "$1" ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +VERSION="$1" +RUST_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# Writes the output of a filter over a file back to that file. +rewrite() { + local file="$1" + shift + "$@" < "$file" > "$file.tmp" + mv "$file.tmp" "$file" +} + +find "$RUST_ROOT" -name Cargo.toml -not -path '*/target/*' -not -path '*/build/*' -print \ + | while IFS= read -r manifest; do + rewrite "$manifest" sed -E \ + -e "s/^version = \"[^\"]*\"$/version = \"$VERSION\"/" \ + -e "s/(path = \"[^\"]*\", version = )\"[^\"]*\"/\1\"$VERSION\"/" + done + +rewrite "$RUST_ROOT/Cargo.lock" awk -v version="$VERSION" ' + BEGIN { RS = "" } + /^\[\[package\]\]/ && !/\nsource = / { + sub(/\nversion = "[^"]*"/, "\nversion = \"" version "\"") + } + { printf "%s%s", (NR > 1 ? "\n\n" : ""), $0 } + END { print "" } +' diff --git a/settings.gradle.kts b/settings.gradle.kts index 2681148095d6..67fb756dc7f4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -366,6 +366,7 @@ include(":sdks:python:test-suites:tox:py314") include(":sdks:python:test-suites:xlang") include(":sdks:typescript") include(":sdks:typescript:container") +include(":sdks:rust") include(":vendor:grpc-1_69_0") include(":vendor:calcite-1_40_0") include(":vendor:guava-32_1_2-jre")