test(monorepo): [WIP] adds adhoc package selection for CI system tests

.kokoro/system.sh

@@ -246,6 +246,54 @@ for path in `find 'packages' \
   fi
 done
 
+# --- Ad-hoc Testing Integration ---
+TRIGGER_ADHOC="false"
+if [[ -n "${KOKORO_GITHUB_PULL_REQUEST_NUMBER}" ]]; then
+    echo "Checking for adhoc test label on PR #${KOKORO_GITHUB_PULL_REQUEST_NUMBER}..."
+    headers=(-H "User-Agent: Kokoro")
+    if [[ -n "${GITHUB_TOKEN:-${GH_TOKEN}}" ]]; then
+        headers+=(-H "Authorization: token ${GITHUB_TOKEN:-${GH_TOKEN}}")
+    fi
+    # Hardened curl call with || true to prevent script termination if network fails
+    LABELS_JSON=$(curl -s "${headers[@]}" "https://api.github.com/repos/googleapis/google-cloud-python/issues/${KOKORO_GITHUB_PULL_REQUEST_NUMBER}/labels" || echo "[]")
+
+    # For this prototype:
+    # we use a small inline Python snippet here because parsing JSON in pure Bash is difficult/error-prone,
+    # and we cannot guarantee that tools like 'jq' or 'gh' are installed in the test environment.
+    # Python and its built-in 'json' module are guaranteed to be available in this repository.
+    IS_ADHOC=$(python3 -c "
+import json
+import sys
+try:
+    labels = json.loads(sys.argv[1])
+    if isinstance(labels, list):
+        if any(isinstance(l, dict) and l.get('name') == 'test:adhoc' for l in labels):
+            print('true')
+except Exception:
+    pass
+" "$LABELS_JSON")
+
+    if [[ "$IS_ADHOC" == "true" ]]; then
+        TRIGGER_ADHOC="true"
+        echo "Adhoc test label 'test:adhoc' found!"
+    else
+        echo "Adhoc test label not found or error occurred."
+    fi
+fi
+
+if [[ "$TRIGGER_ADHOC" == "true" ]]; then
+    echo "Running ad-hoc package selection..."
+    source ci/adhoc/adhoc_test_runner.sh
+
+    echo "Deduplicating packages..."
+    # Portable deduplication avoiding 'declare -A' (compatible with older Bash)
+    COMBINED=$(printf "%s\n" "${PACKAGES_TO_TEST[@]}" $ADHOC_PACKAGES | sort -u | grep -v '^$' || true)
+    PACKAGES_TO_TEST=($COMBINED)
+
+    echo "Combined packages to test: ${PACKAGES_TO_TEST[*]}"
+fi
+# --- End Ad-hoc Testing Integration ---
+
 # Parallel Execution Logic
 MAX_JOBS=${MAX_JOBS:-4}
 
@@ -284,8 +332,10 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \
         log_file="$LOG_DIR/$pkg.log"
       fi
 
-      # Run test; if it fails, create a .failed file to signal failure to the reaper
-      run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed"
+      # EXPERIMENTAL: Added timeout to prevent hanging tests from blocking the whole run.
+      # This is for experimentation only and will be removed in the final design.
+      # We must use 'bash -c' because timeout expects an executable, not an exported function.
+      timeout 45m bash -c 'run_package_test "$1"' _ "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed"
     ' _ "{}"
 
 reap_parallel_results || RETVAL=1

ci/adhoc/.package_groups.txt

@@ -0,0 +1,4 @@
+handwritten: google-cloud-translate
+handwritten: google-cloud-logging
+core: google-api-core
+core: google-cloud-core

ci/adhoc/.standalone_package_list.txt

@@ -0,0 +1,3 @@
+package: google-cloud-logging
+package: google-cloud-dns
+group: handwritten

ci/adhoc/adhoc_test_runner.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Script to determine ad-hoc packages to test.
+# This script is intended to be sourced from main test scripts.
+#
+# Ensure we are in the project root if called directly,
+# but usually this is sourced and CWD is already project root.
+# For safety, we can use script location but if sourced $0 might be the parent script.
+# Let's assume CWD is project root as per system.sh behavior.
+
+ADHOC_DIR="ci/adhoc"
+STANDALONE_LIST="${ADHOC_DIR}/.standalone_package_list.txt"
+GROUPS_FILE="${ADHOC_DIR}/.package_groups.txt"
+
+if [[ ! -f "$STANDALONE_LIST" ]]; then
+    echo "Warning: $STANDALONE_LIST not found."
+    return 0 2>/dev/null || exit 0
+fi
+
+if [[ ! -f "$GROUPS_FILE" ]]; then
+    echo "Warning: $GROUPS_FILE not found."
+    return 0 2>/dev/null || exit 0
+fi
+
+# Grab individual packages
+adhoc_packages=$(grep "^package:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs || true)
+
+# Grab requested groups
+requested_groups=$(grep "^group:" "$STANDALONE_LIST" | cut -d':' -f2 | xargs || true)
+
+# Expand groups
+for group in $requested_groups; do
+    group_pkgs=$(grep "^$group:" "$GROUPS_FILE" | cut -d':' -f2 | xargs || true)
+    adhoc_packages="$adhoc_packages $group_pkgs"
+done
+
+# Convert to unique list (deduplicate our adhoc packages)
+ADHOC_PACKAGES=$(echo "$adhoc_packages" | tr ' ' '\n' | sort -u | xargs)
+
+export ADHOC_PACKAGES

packages/google-cloud-speech/google/cloud/speech_v1/services/speech/client.py

@@ -13,6 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# EXPERIMENTAL: This is a temporary change to trigger Kokoro system tests.
+# Because Kokoro is currently configured to only watch specific package paths (like packages/google-cloud-speech/.*),
+# we must touch a file in one of those paths to wake it up.
+#
+# This file is GAPIC_AUTO, and this specific file is NOT one of the 5 tracked files (setup.py, etc.)
+# in system.sh, so this change will NOT trigger tests for google-cloud-speech itself.
+#
+# If this ad-hoc testing prototype proves successful, we will update the internal Kokoro
+# JobConfigs to watch 'ci/adhoc/.*' instead, eliminating the need for this workaround.
+
 import json
 import logging as std_logging
 import os
@@ -45,9 +55,8 @@
 from google.auth.exceptions import MutualTLSChannelError  # type: ignore
 from google.auth.transport import mtls  # type: ignore
 from google.auth.transport.grpc import SslCredentials  # type: ignore
-from google.oauth2 import service_account  # type: ignore
-
 from google.cloud.speech_v1 import gapic_version as package_version
+from google.oauth2 import service_account  # type: ignore
 
 try:
     OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None]
@@ -67,9 +76,8 @@
 import google.api_core.operation_async as operation_async  # type: ignore
 import google.protobuf.duration_pb2 as duration_pb2  # type: ignore
 import google.rpc.status_pb2 as status_pb2  # type: ignore
-from google.longrunning import operations_pb2  # type: ignore
-
 from google.cloud.speech_v1.types import cloud_speech
+from google.longrunning import operations_pb2  # type: ignore
 
 from .transports.base import DEFAULT_CLIENT_INFO, SpeechTransport
 from .transports.grpc import SpeechGrpcTransport

packages/google-resumable-media/tests/system/requests/test_download.py

@@ -21,15 +21,13 @@
 
 import google.auth  # type: ignore
 import google.auth.transport.requests as tr_requests  # type: ignore
-import pytest  # type: ignore
-
-from google.resumable_media import common
 import google.resumable_media.requests as resumable_requests
-from google.resumable_media import _helpers
-from google.resumable_media.requests import _request_helpers
 import google.resumable_media.requests.download as download_mod
-from tests.system import utils
+import pytest  # type: ignore
+from google.resumable_media import _helpers, common
+from google.resumable_media.requests import _request_helpers
 
+from tests.system import utils
 
 CURR_DIR = os.path.dirname(os.path.realpath(__file__))
 DATA_DIR = os.path.join(CURR_DIR, "..", "..", "data")
@@ -270,6 +268,7 @@ def _read_response_content(response):
 
     @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
     def test_download_full(self, add_files, authorized_transport, checksum):
+        assert False, "Intentional failure to verify ad-hoc testing behavior"
         for info in ALL_FILES:
             actual_contents = self._get_contents(info)
             blob_name = get_blob_name(info)