From 4f980bdbf6e7174693ceea251e91ab9e8bd30b49 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 07:38:55 -0400 Subject: [PATCH 01/20] feat: implement ad-hoc package selection for CI/CD --- .kokoro/system.sh | 45 +++++++++++++++++++++++++++ ci/adhoc/.package_groups.txt | 4 +++ ci/adhoc/.standalone_package_list.txt | 3 ++ ci/adhoc/adhoc_test_runner.sh | 40 ++++++++++++++++++++++++ ci/run_conditional_tests.sh | 39 +++++++++++++++++++++++ 5 files changed, 131 insertions(+) create mode 100644 ci/adhoc/.package_groups.txt create mode 100644 ci/adhoc/.standalone_package_list.txt create mode 100755 ci/adhoc/adhoc_test_runner.sh diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 1d4414b8a92a..1162b93d8545 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -246,6 +246,51 @@ for path in `find 'packages' \ fi done +# --- Ad-hoc Testing Integration --- +TRIGGER_ADHOC="false" +if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]]; then + echo "Checking for adhoc test label on PR #${KOKORO_GITHUB_PULL_REQUEST_NUMBER}..." + LABELS_JSON=$(curl -s -H "User-Agent: Kokoro" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels") + + IS_ADHOC=$(python3 -c " +import json +import sys +try: + labels = json.loads(sys.argv[1]) + if any(l.get('name') == 'test:adhoc' for l in labels): + print('true') +except Exception: + pass +" "$LABELS_JSON") + + if [[ "$IS_ADHOC" == "true" ]]; then + TRIGGER_ADHOC="true" + echo "Adhoc test label 'test:adhoc' found!" + fi +fi + +if [[ "$TRIGGER_ADHOC" == "true" ]]; then + echo "Running ad-hoc package selection..." + source ci/adhoc/adhoc_test_runner.sh + + declare -A unique_packages + for pkg in "${PACKAGES_TO_TEST[@]}"; do + unique_packages["$pkg"]=1 + done + + for pkg in $ADHOC_PACKAGES; do + unique_packages["$pkg"]=1 + done + + PACKAGES_TO_TEST=() + for pkg in "${!unique_packages[@]}"; do + PACKAGES_TO_TEST+=("$pkg") + done + + echo "Combined packages to test: ${PACKAGES_TO_TEST[*]}" +fi +# --- End Ad-hoc Testing Integration --- + # Parallel Execution Logic MAX_JOBS=${MAX_JOBS:-4} diff --git a/ci/adhoc/.package_groups.txt b/ci/adhoc/.package_groups.txt new file mode 100644 index 000000000000..2ea28bb05dcf --- /dev/null +++ b/ci/adhoc/.package_groups.txt @@ -0,0 +1,4 @@ +handwritten: google-cloud-bigquery +handwritten: google-cloud-logging +core: google-api-core +core: google-cloud-core diff --git a/ci/adhoc/.standalone_package_list.txt b/ci/adhoc/.standalone_package_list.txt new file mode 100644 index 000000000000..ecbd2dfe04af --- /dev/null +++ b/ci/adhoc/.standalone_package_list.txt @@ -0,0 +1,3 @@ +package: google-cloud-logging +package: google-cloud-bigtable +group: handwritten diff --git a/ci/adhoc/adhoc_test_runner.sh b/ci/adhoc/adhoc_test_runner.sh new file mode 100755 index 000000000000..2fb1d5eedfa8 --- /dev/null +++ b/ci/adhoc/adhoc_test_runner.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Script to determine ad-hoc packages to test. +# This script is intended to be sourced from main test scripts. + +# Ensure we are in the project root if called directly, +# but usually this is sourced and CWD is already project root. +# For safety, we can use script location but if sourced $0 might be the parent script. +# Let's assume CWD is project root as per system.sh behavior. + +ADHOC_DIR="ci/adhoc" +STANDALONE_LIST="${ADHOC_DIR}/.standalone_package_list.txt" +GROUPS_FILE="${ADHOC_DIR}/.package_groups.txt" + +if [[ ! -f "$STANDALONE_LIST" ]]; then + echo "Warning: $STANDALONE_LIST not found." + return 0 2>/dev/null || exit 0 +fi + +if [[ ! -f "$GROUPS_FILE" ]]; then + echo "Warning: $GROUPS_FILE not found." + return 0 2>/dev/null || exit 0 +fi + +# Grab individual packages +adhoc_packages=$(grep "^package:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs) + +# Grab requested groups +requested_groups=$(grep "^group:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs) + +# Expand groups +for group in $requested_groups; do + group_pkgs=$(grep "^$group:" "$GROUPS_FILE" | cut -d':' -f2 | xargs) + adhoc_packages="$adhoc_packages $group_pkgs" +done + +# Convert to unique list (deduplicate our adhoc packages) +ADHOC_PACKAGES=$(echo $adhoc_packages | tr ' ' '\n' | sort -u | xargs) + +export ADHOC_PACKAGES diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh index 9b8eaee52e5b..aa1cb85acdcc 100755 --- a/ci/run_conditional_tests.sh +++ b/ci/run_conditional_tests.sh @@ -82,6 +82,34 @@ subdirs=( packages ) +# --- Ad-hoc Testing Integration --- +TRIGGER_ADHOC="false" +if [[ -n "${GITHUB_EVENT_PATH}" ]]; then + IS_ADHOC=$(python3 -c " +import json +import os +import sys +try: + with open(os.environ['GITHUB_EVENT_PATH']) as f: + event = json.load(f) + labels = event.get('pull_request', {}).get('labels', []) + if any(l.get('name') == 'test:adhoc' for l in labels): + print('true') +except Exception: + pass +") + if [[ "$IS_ADHOC" == "true" ]]; then + TRIGGER_ADHOC="true" + echo "Adhoc test label 'test:adhoc' found!" + fi +fi + +if [[ "$TRIGGER_ADHOC" == "true" ]]; then + echo "Running ad-hoc package selection..." + source ci/adhoc/adhoc_test_runner.sh +fi +# --- End Ad-hoc Testing Integration --- + RETVAL=0 for subdir in ${subdirs[@]}; do @@ -103,6 +131,17 @@ for subdir in ${subdirs[@]}; do # If GIT_DIFF_ARG is empty, run all the tests. should_test=true fi + + # Ad-hoc check + pkg_name=${d#packages/} + pkg_name=${pkg_name%%/*} + if [[ "$TRIGGER_ADHOC" == "true" ]]; then + if [[ " $ADHOC_PACKAGES " =~ " $pkg_name " ]]; then + echo "Ad-hoc test requested for ${pkg_name}" + should_test=true + fi + fi + if [ "${should_test}" = true ]; then echo "running test in ${d}" pushd ${d} From 59b51027f3a92a59da04679c4c9f067df408af00 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 07:51:30 -0400 Subject: [PATCH 02/20] chore: revert ad-hoc integration in run_conditional_tests.sh to focus on system tests only --- ci/run_conditional_tests.sh | 39 ------------------------------------- 1 file changed, 39 deletions(-) diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh index aa1cb85acdcc..9b8eaee52e5b 100755 --- a/ci/run_conditional_tests.sh +++ b/ci/run_conditional_tests.sh @@ -82,34 +82,6 @@ subdirs=( packages ) -# --- Ad-hoc Testing Integration --- -TRIGGER_ADHOC="false" -if [[ -n "${GITHUB_EVENT_PATH}" ]]; then - IS_ADHOC=$(python3 -c " -import json -import os -import sys -try: - with open(os.environ['GITHUB_EVENT_PATH']) as f: - event = json.load(f) - labels = event.get('pull_request', {}).get('labels', []) - if any(l.get('name') == 'test:adhoc' for l in labels): - print('true') -except Exception: - pass -") - if [[ "$IS_ADHOC" == "true" ]]; then - TRIGGER_ADHOC="true" - echo "Adhoc test label 'test:adhoc' found!" - fi -fi - -if [[ "$TRIGGER_ADHOC" == "true" ]]; then - echo "Running ad-hoc package selection..." - source ci/adhoc/adhoc_test_runner.sh -fi -# --- End Ad-hoc Testing Integration --- - RETVAL=0 for subdir in ${subdirs[@]}; do @@ -131,17 +103,6 @@ for subdir in ${subdirs[@]}; do # If GIT_DIFF_ARG is empty, run all the tests. should_test=true fi - - # Ad-hoc check - pkg_name=${d#packages/} - pkg_name=${pkg_name%%/*} - if [[ "$TRIGGER_ADHOC" == "true" ]]; then - if [[ " $ADHOC_PACKAGES " =~ " $pkg_name " ]]; then - echo "Ad-hoc test requested for ${pkg_name}" - should_test=true - fi - fi - if [ "${should_test}" = true ]; then echo "running test in ${d}" pushd ${d} From 55baa28f2967b19f190d4e17b02f12129ab47794 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 07:54:45 -0400 Subject: [PATCH 03/20] docs: add comment explaining inline python usage in system.sh --- .kokoro/system.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 1162b93d8545..1b6508122bb0 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -252,6 +252,9 @@ if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]]; then echo "Checking for adhoc test label on PR #${KOKORO_GITHUB_PULL_REQUEST_NUMBER}..." LABELS_JSON=$(curl -s -H "User-Agent: Kokoro" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels") + # We use a small inline Python snippet here because parsing JSON in pure Bash is difficult/error-prone, + # and we cannot guarantee that tools like 'jq' or 'gh' are installed in the test environment. + # Python and its built-in 'json' module are guaranteed to be available in this repository. IS_ADHOC=$(python3 -c " import json import sys From bc3081b67c3308e410bbc41fdb7f1233aa4ecf1f Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 07:58:39 -0400 Subject: [PATCH 04/20] docs: tweak comment explaining inline python usage --- .kokoro/system.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 1b6508122bb0..fd7f66fea9d1 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -252,7 +252,8 @@ if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]]; then echo "Checking for adhoc test label on PR #${KOKORO_GITHUB_PULL_REQUEST_NUMBER}..." LABELS_JSON=$(curl -s -H "User-Agent: Kokoro" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels") - # We use a small inline Python snippet here because parsing JSON in pure Bash is difficult/error-prone, + # For this prototype: + # we use a small inline Python snippet here because parsing JSON in pure Bash is difficult/error-prone, # and we cannot guarantee that tools like 'jq' or 'gh' are installed in the test environment. # Python and its built-in 'json' module are guaranteed to be available in this repository. IS_ADHOC=$(python3 -c " From b0d077ef10eb80c1db278bcbeb6dbcf31efeb886 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 08:38:53 -0400 Subject: [PATCH 05/20] fix: make grep commands safe and quote variables in adhoc_test_runner.sh --- ci/adhoc/adhoc_test_runner.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/adhoc/adhoc_test_runner.sh b/ci/adhoc/adhoc_test_runner.sh index 2fb1d5eedfa8..8a658da9013f 100755 --- a/ci/adhoc/adhoc_test_runner.sh +++ b/ci/adhoc/adhoc_test_runner.sh @@ -23,18 +23,18 @@ if [[ ! -f "$GROUPS_FILE" ]]; then fi # Grab individual packages -adhoc_packages=$(grep "^package:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs) +adhoc_packages=$(grep "^package:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs || true) # Grab requested groups -requested_groups=$(grep "^group:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs) +requested_groups=$(grep "^group:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs || true) # Expand groups for group in $requested_groups; do - group_pkgs=$(grep "^$group:" "$GROUPS_FILE" | cut -d':' -f2 | xargs) + group_pkgs=$(grep "^$group:" "$GROUPS_FILE" | cut -d':' -f2 | xargs || true) adhoc_packages="$adhoc_packages $group_pkgs" done # Convert to unique list (deduplicate our adhoc packages) -ADHOC_PACKAGES=$(echo $adhoc_packages | tr ' ' '\n' | sort -u | xargs) +ADHOC_PACKAGES=$(echo "$adhoc_packages" | tr ' ' '\n' | sort -u | xargs) export ADHOC_PACKAGES From e710e45906b31ece35719a4f0fc12533559283cb Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 08:39:05 -0400 Subject: [PATCH 06/20] fix: add auth token to curl and harden inline python in system.sh --- .kokoro/system.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index fd7f66fea9d1..4fbd85ddff33 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -250,7 +250,11 @@ done TRIGGER_ADHOC="false" if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]]; then echo "Checking for adhoc test label on PR #${KOKORO_GITHUB_PULL_REQUEST_NUMBER}..." - LABELS_JSON=$(curl -s -H "User-Agent: Kokoro" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels") + headers=(-H "User-Agent: Kokoro") + if [[ -n "${GITHUB_TOKEN:-${GH_TOKEN}}" ]]; then + headers+=(-H "Authorization: token ${GITHUB_TOKEN:-${GH_TOKEN}}") + fi + LABELS_JSON=$(curl -s "${headers[@]}" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels") # For this prototype: # we use a small inline Python snippet here because parsing JSON in pure Bash is difficult/error-prone, @@ -261,8 +265,9 @@ import json import sys try: labels = json.loads(sys.argv[1]) - if any(l.get('name') == 'test:adhoc' for l in labels): - print('true') + if isinstance(labels, list): + if any(isinstance(l, dict) and l.get('name') == 'test:adhoc' for l in labels): + print('true') except Exception: pass " "$LABELS_JSON") From fbab28a636604e96a16c84df434729e1e79f0585 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 29 Jul 2026 09:26:02 -0400 Subject: [PATCH 07/20] Apply suggestion from @chalmerlowe --- ci/adhoc/adhoc_test_runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/adhoc/adhoc_test_runner.sh b/ci/adhoc/adhoc_test_runner.sh index 8a658da9013f..a7e0c10f880b 100755 --- a/ci/adhoc/adhoc_test_runner.sh +++ b/ci/adhoc/adhoc_test_runner.sh @@ -2,7 +2,7 @@ # Script to determine ad-hoc packages to test. # This script is intended to be sourced from main test scripts. - +# # Ensure we are in the project root if called directly, # but usually this is sourced and CWD is already project root. # For safety, we can use script location but if sourced $0 might be the parent script. From 4bda578c41ff99a6d861f0030fcfb6c5f7faceca Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 09:33:05 -0400 Subject: [PATCH 08/20] fix: harden ad-hoc integration in system.sh against silent failures --- .kokoro/system.sh | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 4fbd85ddff33..628ce7a3a3c9 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -254,7 +254,8 @@ if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]]; then if [[ -n "${GITHUB_TOKEN:-${GH_TOKEN}}" ]]; then headers+=(-H "Authorization: token ${GITHUB_TOKEN:-${GH_TOKEN}}") fi - LABELS_JSON=$(curl -s "${headers[@]}" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels") + # Hardened curl call with || true to prevent script termination if network fails + LABELS_JSON=$(curl -s "${headers[@]}" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels" || echo "[]") # For this prototype: # we use a small inline Python snippet here because parsing JSON in pure Bash is difficult/error-prone, @@ -275,6 +276,8 @@ except Exception: if [[ "$IS_ADHOC" == "true" ]]; then TRIGGER_ADHOC="true" echo "Adhoc test label 'test:adhoc' found!" + else + echo "Adhoc test label not found or error occurred." fi fi @@ -282,19 +285,10 @@ if [[ "$TRIGGER_ADHOC" == "true" ]]; then echo "Running ad-hoc package selection..." source ci/adhoc/adhoc_test_runner.sh - declare -A unique_packages - for pkg in "${PACKAGES_TO_TEST[@]}"; do - unique_packages["$pkg"]=1 - done - - for pkg in $ADHOC_PACKAGES; do - unique_packages["$pkg"]=1 - done - - PACKAGES_TO_TEST=() - for pkg in "${!unique_packages[@]}"; do - PACKAGES_TO_TEST+=("$pkg") - done + echo "Deduplicating packages..." + # Portable deduplication avoiding 'declare -A' (compatible with older Bash) + COMBINED=$(printf "%s\n" "${PACKAGES_TO_TEST[@]}" $ADHOC_PACKAGES | sort -u | grep -v '^$' || true) + PACKAGES_TO_TEST=($COMBINED) echo "Combined packages to test: ${PACKAGES_TO_TEST[*]}" fi From 4424735289f3ef4b80a4aad74fcebf7f673af17d Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 10:04:54 -0400 Subject: [PATCH 09/20] chore: add experimental comment to trigger kokoro --- .../cloud/speech_v1/services/speech/client.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/google-cloud-speech/google/cloud/speech_v1/services/speech/client.py b/packages/google-cloud-speech/google/cloud/speech_v1/services/speech/client.py index 0ac38ec19cdf..4639cae41d8e 100644 --- a/packages/google-cloud-speech/google/cloud/speech_v1/services/speech/client.py +++ b/packages/google-cloud-speech/google/cloud/speech_v1/services/speech/client.py @@ -13,6 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# EXPERIMENTAL: This is a temporary change to trigger Kokoro system tests. +# Because Kokoro is currently configured to only watch specific package paths (like packages/google-cloud-speech/.*), +# we must touch a file in one of those paths to wake it up. +# +# This file is GAPIC_AUTO, and this specific file is NOT one of the 5 tracked files (setup.py, etc.) +# in system.sh, so this change will NOT trigger tests for google-cloud-speech itself. +# +# If this ad-hoc testing prototype proves successful, we will update the internal Kokoro +# JobConfigs to watch 'ci/adhoc/.*' instead, eliminating the need for this workaround. + import json import logging as std_logging import os @@ -45,9 +55,8 @@ from google.auth.exceptions import MutualTLSChannelError # type: ignore from google.auth.transport import mtls # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore -from google.oauth2 import service_account # type: ignore - from google.cloud.speech_v1 import gapic_version as package_version +from google.oauth2 import service_account # type: ignore try: OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] @@ -67,9 +76,8 @@ import google.api_core.operation_async as operation_async # type: ignore import google.protobuf.duration_pb2 as duration_pb2 # type: ignore import google.rpc.status_pb2 as status_pb2 # type: ignore -from google.longrunning import operations_pb2 # type: ignore - from google.cloud.speech_v1.types import cloud_speech +from google.longrunning import operations_pb2 # type: ignore from .transports.base import DEFAULT_CLIENT_INFO, SpeechTransport from .transports.grpc import SpeechGrpcTransport From f38b42c90151034158fae59132740e80a303eab4 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 13:57:05 -0400 Subject: [PATCH 10/20] chore: add experimental timeout to parallel test execution in system.sh --- .kokoro/system.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 628ce7a3a3c9..41caaedba0c6 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -332,8 +332,9 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ log_file="$LOG_DIR/$pkg.log" fi - # Run test; if it fails, create a .failed file to signal failure to the reaper - run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" + # EXPERIMENTAL: Added timeout to prevent hanging tests from blocking the whole run. + # This is for experimentation only and will be removed in the final design. + timeout 45m run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" ' _ "{}" reap_parallel_results || RETVAL=1 From 962bc9e08bd0a1212ae3ebb50e9087ae7c97f099 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 13:57:11 -0400 Subject: [PATCH 11/20] chore: replace heavy packages (bigquery, bigtable) with lighter ones in adhoc configs --- ci/adhoc/.package_groups.txt | 2 +- ci/adhoc/.standalone_package_list.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/adhoc/.package_groups.txt b/ci/adhoc/.package_groups.txt index 2ea28bb05dcf..fe24e6ba4c96 100644 --- a/ci/adhoc/.package_groups.txt +++ b/ci/adhoc/.package_groups.txt @@ -1,4 +1,4 @@ -handwritten: google-cloud-bigquery +handwritten: google-cloud-translate handwritten: google-cloud-logging core: google-api-core core: google-cloud-core diff --git a/ci/adhoc/.standalone_package_list.txt b/ci/adhoc/.standalone_package_list.txt index ecbd2dfe04af..31760352449c 100644 --- a/ci/adhoc/.standalone_package_list.txt +++ b/ci/adhoc/.standalone_package_list.txt @@ -1,3 +1,3 @@ package: google-cloud-logging -package: google-cloud-bigtable +package: google-cloud-dns group: handwritten From 7ca31c92736faa233f1917caaa5e4c2d3693ff36 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 14:31:45 -0400 Subject: [PATCH 12/20] fix: wrap run_package_test in bash -c for timeout command --- .kokoro/system.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 41caaedba0c6..1f54334b2665 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -334,7 +334,8 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ # EXPERIMENTAL: Added timeout to prevent hanging tests from blocking the whole run. # This is for experimentation only and will be removed in the final design. - timeout 45m run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" + # We must use 'bash -c' because timeout expects an executable, not an exported function. + timeout 45m bash -c 'run_package_test "$1"' _ "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" ' _ "{}" reap_parallel_results || RETVAL=1 From 801d0f6263d46b5cabb7adfa8ac439ad67a83191 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 29 Jul 2026 17:24:23 -0400 Subject: [PATCH 13/20] chore: inject intentional failure in google-resumable-media to test ad-hoc behavior --- .../tests/system/requests/test_download.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/google-resumable-media/tests/system/requests/test_download.py b/packages/google-resumable-media/tests/system/requests/test_download.py index f0bd6c7e30e1..5d0a517e9e97 100644 --- a/packages/google-resumable-media/tests/system/requests/test_download.py +++ b/packages/google-resumable-media/tests/system/requests/test_download.py @@ -21,15 +21,13 @@ import google.auth # type: ignore import google.auth.transport.requests as tr_requests # type: ignore -import pytest # type: ignore - -from google.resumable_media import common import google.resumable_media.requests as resumable_requests -from google.resumable_media import _helpers -from google.resumable_media.requests import _request_helpers import google.resumable_media.requests.download as download_mod -from tests.system import utils +import pytest # type: ignore +from google.resumable_media import _helpers, common +from google.resumable_media.requests import _request_helpers +from tests.system import utils CURR_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(CURR_DIR, "..", "..", "data") @@ -270,6 +268,7 @@ def _read_response_content(response): @pytest.mark.parametrize("checksum", ["md5", "crc32c", None]) def test_download_full(self, add_files, authorized_transport, checksum): + assert False, "Intentional failure to verify ad-hoc testing behavior" for info in ALL_FILES: actual_contents = self._get_contents(info) blob_name = get_blob_name(info) From 3ac8fa5fd8e632769e33f780622b2547c8850f26 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 30 Jul 2026 05:07:11 -0400 Subject: [PATCH 14/20] chore: break setup.py in google-resumable-media to guarantee failure --- packages/google-resumable-media/setup.py | 1 + .../tests/system/requests/test_download.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/google-resumable-media/setup.py b/packages/google-resumable-media/setup.py index 44606bbfb63c..c20c30450da2 100644 --- a/packages/google-resumable-media/setup.py +++ b/packages/google-resumable-media/setup.py @@ -16,6 +16,7 @@ import setuptools +raise RuntimeError("Intentional breakage to verify ad-hoc testing behavior") PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__)) diff --git a/packages/google-resumable-media/tests/system/requests/test_download.py b/packages/google-resumable-media/tests/system/requests/test_download.py index 5d0a517e9e97..260adaa94f9a 100644 --- a/packages/google-resumable-media/tests/system/requests/test_download.py +++ b/packages/google-resumable-media/tests/system/requests/test_download.py @@ -268,7 +268,6 @@ def _read_response_content(response): @pytest.mark.parametrize("checksum", ["md5", "crc32c", None]) def test_download_full(self, add_files, authorized_transport, checksum): - assert False, "Intentional failure to verify ad-hoc testing behavior" for info in ALL_FILES: actual_contents = self._get_contents(info) blob_name = get_blob_name(info) From 2269f4e842f86908bce33524e70c7bb9176bf405 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 30 Jul 2026 05:12:40 -0400 Subject: [PATCH 15/20] chore: dump logs for passed packages in system.sh for debugging --- .kokoro/system.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 1f54334b2665..f809a6aae5e0 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -160,8 +160,20 @@ reap_parallel_results() { if [ ${#passed_packages[@]} -gt 0 ]; then echo "" - echo "PASSED PACKAGES:" - printf "%s\n" "${passed_packages[@]}" | sort | sed 's/^/- /' + echo "==================================================" + echo " LOGS FOR PASSED PACKAGES " + echo "==================================================" + for pkg in "${passed_packages[@]}"; do + echo "--------------------------------------------------" + echo "LOGS FOR PASSED PACKAGE: $pkg" + echo "--------------------------------------------------" + if [ -n "$KOKORO_ARTIFACTS_DIR" ] && [ -f "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" ]; then + cat "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" + else + cat "$LOG_DIR/$pkg.log" + fi + echo "" + done fi if [ "$failed_count" -gt 0 ]; then From 4861794b6aa55ce69f5ad271918f4881c18ce940 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 30 Jul 2026 05:22:40 -0400 Subject: [PATCH 16/20] chore: add debug echoes and robustify log dumping in system.sh --- .kokoro/system.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index f809a6aae5e0..2f8cbf7d339f 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -169,8 +169,11 @@ reap_parallel_results() { echo "--------------------------------------------------" if [ -n "$KOKORO_ARTIFACTS_DIR" ] && [ -f "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" ]; then cat "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" - else + elif [ -f "$LOG_DIR/$pkg.log" ]; then cat "$LOG_DIR/$pkg.log" + else + echo "Warning: No log file found for passed package $pkg" + echo "Expected either $KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log or $LOG_DIR/$pkg.log" fi echo "" done @@ -187,8 +190,11 @@ reap_parallel_results() { echo "--------------------------------------------------" if [ -n "$KOKORO_ARTIFACTS_DIR" ] && [ -f "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" ]; then cat "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" - else + elif [ -f "$LOG_DIR/$pkg.log" ]; then cat "$LOG_DIR/$pkg.log" + else + echo "Warning: No log file found for failed package $pkg" + echo "Expected either $KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log or $LOG_DIR/$pkg.log" fi echo "" fi @@ -335,14 +341,16 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ | xargs -P "$MAX_JOBS" -I {} \ bash -c ' pkg="$1" + echo "Processing package: $pkg" # Determine log location: prefer Sponge artifacts directory if available if [ -n "$KOKORO_ARTIFACTS_DIR" ]; then pkg_log_dir="$KOKORO_ARTIFACTS_DIR/$pkg" - mkdir -p "$pkg_log_dir" || { touch "$LOG_DIR/$pkg.failed"; exit 1; } + mkdir -p "$pkg_log_dir" || { echo "Failed to mkdir $pkg_log_dir"; touch "$LOG_DIR/$pkg.failed"; exit 1; } log_file="$pkg_log_dir/sponge_log.log" else log_file="$LOG_DIR/$pkg.log" fi + echo "Log file for $pkg: $log_file" # EXPERIMENTAL: Added timeout to prevent hanging tests from blocking the whole run. # This is for experimentation only and will be removed in the final design. From b2199b3cb83e1a923fde4a845fc0f77a6edff41c Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 30 Jul 2026 05:30:54 -0400 Subject: [PATCH 17/20] fix: resolve nested quoting bug in timeout command --- .kokoro/system.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 2f8cbf7d339f..4d48fca902c7 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -355,7 +355,8 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ # EXPERIMENTAL: Added timeout to prevent hanging tests from blocking the whole run. # This is for experimentation only and will be removed in the final design. # We must use 'bash -c' because timeout expects an executable, not an exported function. - timeout 45m bash -c 'run_package_test "$1"' _ "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" + # Using double quotes to avoid breaking the outer single-quoted script. + timeout 45m bash -c "run_package_test \"\$1\"" _ "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" ' _ "{}" reap_parallel_results || RETVAL=1 From aeac1b1d05a2fc849f194c4f553fe75fe3036220 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 30 Jul 2026 05:39:55 -0400 Subject: [PATCH 18/20] fix: refactor xargs to use -n 1 for robust argument passing --- .kokoro/system.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 4d48fca902c7..c3acae36059e 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -338,7 +338,7 @@ export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR # -P "$MAX_JOBS" controls concurrency # -I {} replaces {} with the package name printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ - | xargs -P "$MAX_JOBS" -I {} \ + | xargs -n 1 -P "$MAX_JOBS" \ bash -c ' pkg="$1" echo "Processing package: $pkg" @@ -357,7 +357,7 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ # We must use 'bash -c' because timeout expects an executable, not an exported function. # Using double quotes to avoid breaking the outer single-quoted script. timeout 45m bash -c "run_package_test \"\$1\"" _ "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" - ' _ "{}" + ' _ reap_parallel_results || RETVAL=1 From 73100e7daa42a7273ea511db0a091b378cfa3109 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 30 Jul 2026 05:48:43 -0400 Subject: [PATCH 19/20] fix: simplify argument passing to bash -c in xargs to avoid positional parameter confusion --- .kokoro/system.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index c3acae36059e..38bc55b947b1 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -340,7 +340,7 @@ export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ | xargs -n 1 -P "$MAX_JOBS" \ bash -c ' - pkg="$1" + pkg="$0" echo "Processing package: $pkg" # Determine log location: prefer Sponge artifacts directory if available if [ -n "$KOKORO_ARTIFACTS_DIR" ]; then @@ -357,7 +357,7 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ # We must use 'bash -c' because timeout expects an executable, not an exported function. # Using double quotes to avoid breaking the outer single-quoted script. timeout 45m bash -c "run_package_test \"\$1\"" _ "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" - ' _ + ' reap_parallel_results || RETVAL=1 From 042227d23916e0907436fc7f8da0904357e576ff Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 30 Jul 2026 06:03:26 -0400 Subject: [PATCH 20/20] fix: resolve catastrophic quoting bug caused by single quotes in comment --- .kokoro/system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 38bc55b947b1..92da612a4ec3 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -354,7 +354,7 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ # EXPERIMENTAL: Added timeout to prevent hanging tests from blocking the whole run. # This is for experimentation only and will be removed in the final design. - # We must use 'bash -c' because timeout expects an executable, not an exported function. + # We must use "bash -c" because timeout expects an executable, not an exported function. # Using double quotes to avoid breaking the outer single-quoted script. timeout 45m bash -c "run_package_test \"\$1\"" _ "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" '