diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 3fb2d0a402..eccd79a3ad 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -306,6 +306,23 @@ blocks: - Add-Content "C:\kafka_2.13-${env:KAFKA_VERSION}\config\server.properties" "`ngroup.share.min.record.lock.duration.ms=1000`ntransaction.state.log.replication.factor=1`ntransaction.state.log.min.isr=1" - $env:PATH = [System.Environment]::GetEnvironmentVariable("PATH", "Machine") + ";" + [System.Environment]::GetEnvironmentVariable("PATH", "User"); $env:KAFKA_CLUSTER_ID = & "C:\kafka_2.13-${env:KAFKA_VERSION}\bin\windows\kafka-storage.bat" random-uuid; & "C:\kafka_2.13-${env:KAFKA_VERSION}\bin\windows\kafka-storage.bat" format --standalone -t $env:KAFKA_CLUSTER_ID -c "C:\kafka_2.13-${env:KAFKA_VERSION}\config\server.properties"; Start-Job -Name kafka-broker -ScriptBlock { param($v) & "C:\kafka_2.13-$v\bin\windows\kafka-server-start.bat" "C:\kafka_2.13-$v\config\server.properties" } -ArgumentList $env:KAFKA_VERSION; Start-Sleep 15; cd tests; Set-Content test.conf "bootstrap.servers=localhost:9092`nbroker.address.family=v4"; Remove-Item Env:TESTS -ErrorAction SilentlyContinue; Remove-Item Env:TESTS_SKIP -ErrorAction SilentlyContinue; $env:RDKAFKA_TEST_CONF = "test.conf"; $env:TESTS_SKIP_BEFORE = "0155"; $env:TEST_BROKER_OS = "windows"; & ..\win32\outdir\v142\Win32\Release\tests.exe; Stop-Job -Name kafka-broker + - name: 'Linux Ubuntu amd64: AWS-LC FIPS build' + dependencies: [] + skip: + # Skip for release tags, we don't want dependency downloads + # to fail the release build. + when: "tag =~ '^v[0-9]\\.'" + task: + agent: + machine: + type: s1-prod-ubuntu24-04-amd64-1 + jobs: + - name: 'Build with AWS-LC FIPS' + commands: + - sudo apt-get update + - sudo apt-get install -y cmake golang-go perl + - packaging/tools/build-aws-lc-fips.sh + - name: 'Linux x64: release artifact docker builds' dependencies: [] run: diff --git a/packaging/tools/aws-lc-fips-check.c b/packaging/tools/aws-lc-fips-check.c new file mode 100644 index 0000000000..a5c4077c1f --- /dev/null +++ b/packaging/tools/aws-lc-fips-check.c @@ -0,0 +1,41 @@ +/* + * Verify that the dependency used by the AWS-LC FIPS CI build is AWS-LC, + * that its FIPS module passes its runtime checks, and that librdkafka can + * initialize SSL with it. + */ + +#include +#include + +#include "rdkafka.h" + +#ifndef OPENSSL_IS_AWSLC +#error "The configured crypto library is not AWS-LC" +#endif + +int main(void) { + rd_kafka_conf_t *conf; + rd_kafka_t *rk; + char errstr[512]; + + if (!FIPS_mode()) + return 1; + + if (!BORINGSSL_integrity_test()) + return 1; + + conf = rd_kafka_conf_new(); + if (rd_kafka_conf_set(conf, "security.protocol", "ssl", errstr, + sizeof(errstr)) != RD_KAFKA_CONF_OK) { + rd_kafka_conf_destroy(conf); + return 1; + } + + rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); + if (!rk) + return 1; + + rd_kafka_destroy(rk); + + return 0; +} diff --git a/packaging/tools/build-aws-lc-fips.sh b/packaging/tools/build-aws-lc-fips.sh new file mode 100755 index 0000000000..db5914cbaf --- /dev/null +++ b/packaging/tools/build-aws-lc-fips.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -euo pipefail + +AWS_LC_FIPS_REF=${AWS_LC_FIPS_REF:-AWS-LC-FIPS-3.3.0} +AWS_LC_FIPS_REPOSITORY=${AWS_LC_FIPS_REPOSITORY:-https://github.com/aws/aws-lc.git} + +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT + +git clone --depth 1 --branch "$AWS_LC_FIPS_REF" \ + "$AWS_LC_FIPS_REPOSITORY" "$work_dir/aws-lc" + +# Warnings from the external dependency must not affect librdkafka's build. +cmake -S "$work_dir/aws-lc" -B "$work_dir/aws-lc-build" \ + -DCMAKE_BUILD_TYPE=Debug \ + "-DCMAKE_C_FLAGS=-ffunction-sections -fdata-sections -fPIC -w" \ + -DCMAKE_INSTALL_PREFIX="$work_dir/aws-lc-install" \ + -DBUILD_LIBSSL=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_TOOL=OFF \ + -DFIPS=1 +cmake --build "$work_dir/aws-lc-build" --parallel +cmake --install "$work_dir/aws-lc-build" + +aws_lc_libcrypto=$(find "$work_dir/aws-lc-install" -name libcrypto.a -print -quit) +if [[ -z "$aws_lc_libcrypto" ]]; then + echo "AWS-LC FIPS build did not produce libcrypto.a" >&2 + exit 1 +fi +aws_lc_lib_dir=$(dirname "$aws_lc_libcrypto") +aws_lc_include_dir="$work_dir/aws-lc-install/include" + +CPPFLAGS="-I$aws_lc_include_dir" \ +CFLAGS="-Werror=implicit-function-declaration" \ +LDFLAGS="-L$aws_lc_lib_dir" \ +PKG_CONFIG_PATH="$aws_lc_lib_dir/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" \ +./configure --enable-ssl --disable-curl --disable-gssapi --disable-zlib \ + --disable-zstd --disable-lz4-ext --disable-regex-ext --enable-devel +make --jobs libs + +cc -I"$aws_lc_include_dir" -I"$PWD/src" \ + "$PWD/packaging/tools/aws-lc-fips-check.c" \ + -L"$PWD/src" -Wl,-rpath,"$PWD/src" -lrdkafka \ + -L"$aws_lc_lib_dir" -Wl,-Bstatic -lcrypto -Wl,-Bdynamic -lpthread -ldl \ + -o "$work_dir/aws-lc-fips-check" +"$work_dir/aws-lc-fips-check" + +if readelf -d src/librdkafka.so.1 | \ + grep -E 'Shared library: \[lib(ssl|crypto)\.so' >/dev/null; then + echo "librdkafka unexpectedly links to a shared OpenSSL library" >&2 + exit 1 +fi + +for symbol in BORINGSSL_integrity_test FIPS_mode HMAC; do + if ! nm src/librdkafka.so.1 | \ + grep -E "[[:space:]]${symbol}$" >/dev/null; then + echo "librdkafka does not contain AWS-LC FIPS symbol $symbol" >&2 + exit 1 + fi +done diff --git a/src/rdkafka_ssl.c b/src/rdkafka_ssl.c index 35c1fac7f8..6186addb8a 100644 --- a/src/rdkafka_ssl.c +++ b/src/rdkafka_ssl.c @@ -45,6 +45,7 @@ #pragma comment(lib, "libssl.lib") #endif +#include #include #include