Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,7 @@ build:windows --google_default_credentials=true
build:macos --remote_cache=https://storage.googleapis.com/macos-cel-python-remote-cache
build:macos --google_default_credentials=true

# Silence deprecation warnings from external dependencies (Linux and macOS)
build:linux --cxxopt=-Wno-deprecated-declarations
build:macos --cxxopt=-Wno-deprecated-declarations

5 changes: 5 additions & 0 deletions release/kokoro/release_linux.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,8 @@

build_file: "cel-python/release/kokoro/release_linux.sh"
timeout_mins: 120

container_properties {
docker_image: "us-central1-docker.pkg.dev/kokoro-container-bakery/kokoro/ubuntu/ubuntu2204/ktcb:current"
docker_sibling_containers: true
}
171 changes: 162 additions & 9 deletions release/kokoro/release_linux.sh
Original file line number Diff line number Diff line change
@@ -1,21 +1,167 @@
#!/bin/bash
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

# Avoid virtualenv/pip trying to download/upgrade tools from PyPI on host
export VIRTUALENV_NO_DOWNLOAD=1
export PIP_DISABLE_PIP_VERSION_CHECK=1

# Pass these environment variables to the cibuildwheel Docker container
export CIBW_ENVIRONMENT="VIRTUALENV_NO_DOWNLOAD=1 PIP_DISABLE_PIP_VERSION_CHECK=1"
export CIBW_DEPENDENCY_VERSIONS="latest"
export CIBW_CONTAINER_ENGINE_EXTRA_ARGS="--network=host"

# If running locally (not on Kokoro), authenticate with gcloud.
if [ -z "${KOKORO_BUILD_ID}" ]; then
if ! gcloud auth application-default print-access-token --quiet > /dev/null; then
gcloud auth application-default login
fi
fi

pip install -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel
# We use --no-cache-dir to force pip to download packages fresh and bypass the local
# cache. In a sandboxed build environment, writing to the default cache directory
# (~/.cache/pip) can encounter permission/sandbox restrictions or lead to stale
# dependency resolution. Disabling the cache ensures a reliable, reproducible install.
pip install --no-cache-dir -U keyring keyrings.google-artifactregistry-auth twine cibuildwheel

# ==============================================================================
# FUTURE-PROOF RUNTIME PATCHING OF CIBUILDWHEEL
# ==============================================================================
# To run cibuildwheel on Google's sandboxed RBE/Kokoro infrastructure, we must:
# 1. Bypass RBE's stdout proxy buffering deadlock (requires 32KB padding).
# 2. Bypass RBE's stdin EOF deadlock during copy-in (requires 'docker cp'
# since we use disable_host_mount: True in pyproject.toml).
#
# Since cibuildwheel is installed fresh from PyPI on every build (ensuring we get
# the latest security and feature updates), we apply these patches at runtime.
#
# Why this patching strategy is future-proof and safe:
# - Strict Validation: The Python patcher strictly validates that all target
# code blocks exist before applying replacements. If cibuildwheel's internal
# code changes in a future release, the patcher will FAIL LOUDLY and exit the
# build immediately (sys.exit(1)) rather than silently running a broken,
# hanging build.
# - Stable Boundaries: The copy_into patch uses a robust regular expression
# anchored to class method boundaries (def copy_into -> def copy_out). These
# are stable, long-standing internal APIs of cibuildwheel's OCIContainer.
# - Core Protocol Stability: The buffering patches target the core protocol
# used to communicate with the container's persistent bash shell. This
# protocol is fundamental to cibuildwheel and highly unlikely to change.
# ==============================================================================
OCI_PATH=$(python3 -c "import cibuildwheel.oci_container; print(cibuildwheel.oci_container.__file__)")
echo "Patching cibuildwheel at $OCI_PATH..."

cat << 'EOF' > patch_oci.py
import sys
import re

path = sys.argv[1]
with open(path, 'r') as f:
content = f.read()

# 1. Force a 32KB flush at the end of every command execution
target_write = 'printf "%04d%s\\n" $? {end_of_message}'
replacement_write = 'printf "%04d%s\\n%32768s\\n" $? {end_of_message} " "'
if target_write in content:
content = content.replace(target_write, replacement_write)
print("Patched write loop.")
else:
print("ERROR: Could not find write loop target in oci_container.py! The cibuildwheel version might have changed.")
sys.exit(1)

# 2. Read and discard the 32KB padding to keep the stream clean
target_read = """ # add the last line to output, without the footer
output_io.write(line[0:footer_offset])
output_io.flush()
break"""

replacement_read = """ # add the last line to output, without the footer
output_io.write(line[0:footer_offset])
output_io.flush()
# Read and discard the 32KB padding line to clear the stream!
self.bash_stdout.readline()
break"""

if target_read in content:
content = content.replace(target_read, replacement_read)
print("Patched read loop.")
else:
print("ERROR: Could not find read loop target in oci_container.py! The cibuildwheel version might have changed.")
sys.exit(1)

# 3. Patch the entire copy_into method using a unique regex to use native 'docker cp'.
# This bypasses the RBE stdin EOF deadlock when copying the project into the container.
pattern = re.compile(r' def copy_into\(self,.*?\).*?:.*? def copy_out', re.DOTALL)

replacement_copy = """ def copy_into(self, from_path: Path, to_path: PurePath) -> None:
if from_path.is_dir():
self.call(["mkdir", "-p", to_path])
subprocess.run(
f"tar -c {self.host_tar_format} -f - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
shell=True,
check=True,
cwd=from_path,
)
else:
self.call(["mkdir", "-p", to_path.parent])
# Use native docker cp to copy the file, avoiding stdin EOF deadlocks in RBE
subprocess.run(
[
self.engine.name,
"cp",
str(from_path),
f"{self.name}:{to_path}",
],
check=True,
)

def copy_out"""

if pattern.search(content):
content = pattern.sub(replacement_copy, content)
print("Patched copy_into method using unique regex.")
else:
print("ERROR: Could not find copy_into method boundary in oci_container.py! The cibuildwheel version might have changed.")
sys.exit(1)

with open(path, 'w') as f:
f.write(content)

print("Successfully patched oci_container.py!")
EOF

python3 patch_oci.py "$OCI_PATH"
rm patch_oci.py

# Verify that the patched file is syntactically valid Python
echo "Verifying patched oci_container.py syntax..."
python3 -m py_compile "$OCI_PATH" || { echo "ERROR: Patched oci_container.py is corrupted!"; exit 1; }

REPO_DIR=""
TMP_DIR=""
cleanup() {
echo "Cleaning up temporary directories..."
[ -n "${REPO_DIR}" ] && rm -rf "${REPO_DIR}"
[ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}"
}
trap cleanup EXIT

REPO_DIR=$(mktemp -d)
echo "Created temporary directory: ${REPO_DIR}"

# Ensure the temporary directory is removed on script exit
trap 'echo "Cleaning up temporary directory: ${REPO_DIR}"; rm -rf "${REPO_DIR}"' EXIT

if [ "${DRY_RUN}" = "true" ]; then
echo "[DRY RUN] Using local Kokoro clone instead of cloning main."
SRC_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
Expand All @@ -40,24 +186,31 @@ fi
VERSION=${VERSION#v}
echo "Building release for version: ${VERSION}"

TMP_DIR=$(mktemp -d)
# Create the build directory inside the workspace volume (SRC_DIR)
# instead of the ephemeral /tmp, so that the sibling container can
# access it natively via volume propagation
TMP_DIR="${SRC_DIR}/build_area"
mkdir -p "${TMP_DIR}"
echo "Build directory: ${TMP_DIR}"

# Add trap cleanup for TMP_DIR as well
trap 'echo "Cleaning up temporary directories: ${REPO_DIR} ${TMP_DIR}"; rm -rf "${REPO_DIR}" "${TMP_DIR}"' EXIT
export TMPDIR="${TMP_DIR}/tmp"
mkdir -p "${TMPDIR}"

pushd "${TMP_DIR}"

cp -r "${SRC_DIR}"/{*,.*} . 2>/dev/null || true
cp -r "${SRC_DIR}"/release/* . 2>/dev/null || true
rm -rf cel_expr_python/*_test.py

echo "Downloading bazelisk on host..."
curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64
chmod +x bazelisk-linux-amd64

# Check if pyproject.toml exists before running sed
if [ -f pyproject.toml ]; then
sed -i "" "s/\$VERSION/${VERSION}/g" pyproject.toml || sed -i "s/\$VERSION/${VERSION}/g" pyproject.toml
fi

echo "Running cibuildwheel: ${CIBWHEEL_BIN}"
echo "Running cibuildwheel..."
# Default CIBWHEEL_BIN if not set
if [ -z "${CIBWHEEL_BIN}" ]; then
CIBWHEEL_BIN="python3 -m cibuildwheel"
Expand Down
11 changes: 9 additions & 2 deletions release/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,19 @@ exclude = ["codelab*", "conformance*", "custom_ext*", "release*", "testing*", "w

[tool.cibuildwheel]
build = "cp311-* cp312-* cp313-* cp314-*"
skip = "*musllinux* *win32*"
skip = "*musllinux* *win32* *i686*"
test-command = "python {project}/cel_basic_test.py"
build-verbosity = 1

[tool.cibuildwheel.linux]
before-all = "echo 'Installing bazelisk'; curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 && chmod +x bazelisk-linux-amd64 && mv bazelisk-linux-amd64 /usr/local/bin/bazel"
manylinux-x86_64-image = "manylinux_2_28"
container-engine = "docker; disable_host_mount: True"
# Google's internal Kokoro/RBE network uses a secure MITM proxy that resigns HTTPS
# traffic with an internal Google CA. Since the public manylinux container does not
# trust this CA, git fetches for external dependencies (like @cel-cpp) will fail
# with SSL certificate errors. We disable http.sslVerify inside the container to
# bypass this and allow Bazel to fetch SCM dependencies through the proxy.
before-all = "git config --global http.sslVerify false && echo 'Installing bazelisk' && cp {project}/bazelisk-linux-amd64 /usr/local/bin/bazel"

[tool.cibuildwheel.macos]
before-all = "echo 'Installing bazelisk'; brew install bazelisk"
Expand Down
Loading