diff --git a/.bazelrc b/.bazelrc index d90d3dba6..85f5bf35d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -24,6 +24,8 @@ build --@rules_rust//rust/settings:clippy_flags=-Dwarnings # Darwin sandbox reuse otherwise traverses the complete SDK forest per action. build:macos-sandbox --experimental_inmemory_sandbox_stashes +# Reuse shared SDK input hashes when checking the hosted disk action cache. +build:macos-sandbox --experimental_remote_merkle_tree_cache # The LLVM serializer runs on Linux; serialized product targets remain separate. build:wasix-aot --extra_execution_platforms=//build_defs/wasix:llvm_serializer_linux_x86_64 diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index a14de38e5..30511e326 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -957,6 +957,17 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Upload Linux build diagnostics + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: native-linux-build-diagnostics-${{ matrix.platform }} + path: | + ${{ runner.temp }}/*build-capture/diagnostics/ + ${{ runner.temp }}/ci-requests/**/diagnostics/ + if-no-files-found: ignore + retention-days: 7 + - *ci_request_receipts - name: Save bounded standard Bazel caches diff --git a/BUILD.bazel b/BUILD.bazel index 16290b411..0774fe28c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -306,6 +306,7 @@ test_suite( "//build_defs/foundation:toolchain_metadata_test", "//build_defs/host:host_sdk_test", "//build_defs/host:profile_analysis_test", + "//build_defs/host:apple_sdk_inputs_test", "//build_defs/native:host_runtime_test_helpers_test", "//build_defs/native:windows_exports_test", "//build_defs/native:host_config_policy_test", @@ -313,6 +314,7 @@ test_suite( "//build_defs/extensions:forced_header_server_inputs_test", "//build_defs/extensions:forced_header_embedded_inputs_test", "//build_defs/extensions:mobile_host_ios_xcframework_test", + "//build_defs/extensions:mobile_cpp_reuse_test", "//build_defs/extensions:mobile_host_android_arm64_v8a_test", "//build_defs/extensions:mobile_host_android_x86_64_test", "//build_defs/native:postgres_probe_header_test", diff --git a/build_defs/android/README.md b/build_defs/android/README.md index 10c2177c2..bbe9bae04 100644 --- a/build_defs/android/README.md +++ b/build_defs/android/README.md @@ -11,6 +11,24 @@ reuses `rules_android_ndk`'s maintained C/C++ configuration and declares the compiler, headers, target libraries and link tools as action inputs. Small local registration targets keep unrelated platform analysis from acquiring the NDK. +Released native slices and APK native dependencies use +`rules_android`'s `android_split_transition`. The release producer sets its API +floor from the compatibility policy before that split. Packaged archives and +runtime data enter through `android_product_inputs`, which clears the consuming +app's Android split settings and restores the Linux producer context. Executable +tools retain `cfg = "exec"`. This keeps app configuration out of the released +products without copying upstream's native transition or lowering the app's Java +minimum SDK. +`android_native_member` forwards registration and the C++ library together from +that native context, and verifies that both reference the same archive. This +also preserves reuse when the consuming app raises its minimum SDK. + +`//src/examples/native/react-native-expo:android_cpp_reuse_test` compares the actual +selected release archive Files with the Files linked into the example APK for +every declared ABI, at the app's minimum SDK and at a higher app SDK. Its +inventory comes from the existing carrier metadata and selected extensions; +adding an object file does not require updating the test. + Run the acquisition-policy tests and actual cross-compiled ELF checks together: ```sh diff --git a/build_defs/android/app-query-test.bzl b/build_defs/android/app-query-test.bzl index 489b61ea2..55b8cadd0 100644 --- a/build_defs/android/app-query-test.bzl +++ b/build_defs/android/app-query-test.bzl @@ -66,7 +66,7 @@ def _android_min_sdk_test_impl(ctx): for minimum in [24, 26]: configuration = android_app_configuration({}, struct(abi = abi, min_sdk_version = minimum)) asserts.equals(env, minimum, configuration[str(Label("@rules_android//rules:min_sdk_version"))]) - asserts.equals(env, ["//build_defs/android:android_" + platform], configuration["//command_line_option:android_platforms"]) + asserts.equals(env, [str(Label("//build_defs/android:android_" + platform))], configuration["//command_line_option:android_platforms"]) asserts.equals(env, "opt", configuration["//command_line_option:compilation_mode"]) # The maintained dependency transition preserves an incoming SDK # rather than invoking its dormant clamp or lowering the setting. diff --git a/build_defs/android/app.bzl b/build_defs/android/app.bzl index ffcb9cfef..ab1e049ce 100644 --- a/build_defs/android/app.bzl +++ b/build_defs/android/app.bzl @@ -2,6 +2,7 @@ load("@aspect_rules_js//npm:providers.bzl", "NpmPackageInfo") load("@rules_android//providers:providers.bzl", "AndroidSdkInfo", "ApkInfo") +load("@rules_android//rules:android_split_transition.bzl", "android_split_transition") load("@rules_cc//cc:defs.bzl", "cc_import", "cc_library") load("@rules_cc//cc/common:cc_common.bzl", "cc_common") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") @@ -10,6 +11,7 @@ load("//build_defs/foundation:bun.bzl", "BUN_SOURCE_TOOLCHAINS", "BUN_TEST_ENTRY load("//build_defs/extensions:mobile.bzl", "MobileExtensionInfo") load("//build_defs/extensions:final_leaf.bzl", "ExtensionFinalLeafInfo") load("@oliphaunt_metadata//:metadata.bzl", "EXTENSIONS") +load(":configuration.bzl", "ANDROID_TARGET_PLATFORMS", "android_native_inputs", "android_product_inputs") _DESCRIPTORS = {row["sql-name"]: row for row in EXTENSIONS.values()} @@ -274,7 +276,8 @@ def _final_aar_impl(ctx): return [DefaultInfo(files = depset([output]))] android_final_aar = rule(implementation = _final_aar_impl, toolchains = BUN_SOURCE_TOOLCHAINS, - attrs = dict(_RUNNER, **{"source": attr.label(mandatory = True, allow_files = True, cfg = "exec"), + attrs = dict(_RUNNER, **{"source": attr.label(mandatory = True, allow_files = True, cfg = android_product_inputs), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), "kind": attr.string(default = "aar", values = ["aar", "runtime", "seed"]), "catalog_profile": attr.string(default = "standard", values = ["standard", "icu"]), "package": attr.string(), "abi": attr.string(default = "", values = ["", "arm64-v8a", "x86_64"])})) @@ -307,17 +310,38 @@ def _exec_npm_impl(ctx): src = ctx.file.archive, npm_package_store_infos = depset())] android_workspace_npm = rule(implementation = _exec_npm_impl, attrs = { - "archive": attr.label(allow_single_file = [".tgz"], cfg = "exec"), + "archive": attr.label(allow_single_file = [".tgz"], cfg = android_product_inputs), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), "error": attr.string(), "package_name": attr.string(mandatory = True), "version": attr.string(mandatory = True), }) +def _native_member_impl(ctx): + member = ctx.attr.member[0][MobileExtensionInfo] + library = ctx.attr.library[0][CcInfo] + archives = [archive for item in library.linking_context.linker_inputs.to_list() for linked in item.libraries for archive in [linked.static_library, linked.pic_static_library] if archive] + if member.archive not in archives: + fail("Android registration and C++ link must consume the same configured member archive") + return [ctx.attr.member[0][DefaultInfo], member, library] + +# A released native product keeps its compatibility floor when an app raises +# its Java/Dex floor. Forward both registration and the actual C++ link provider +# from the same upstream ABI split, including the library's transitive deps. +android_native_member = rule(implementation = _native_member_impl, cfg = android_native_inputs, attrs = { + "platform": attr.label(mandatory = True), + "member": attr.label(mandatory = True, providers = [MobileExtensionInfo], cfg = android_split_transition), + "library": attr.label(mandatory = True, providers = [CcInfo], cfg = android_split_transition), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), +}) + def _link_receipt_impl(ctx): members, inputs = [], [ctx.file.library, ctx.file.registry] + archives = [] for target in ctx.attr.members: member = target[MobileExtensionInfo] members.append({"archive": member.archive.path, "registration": member.registration.path}) inputs += [member.archive, member.registration] + archives.append(member.archive) nm = [file for file in ctx.files._nm if file.basename == "llvm-nm"] if len(nm) != 1: fail("Static extension proof needs the actual pinned NDK nm tool") @@ -325,7 +349,7 @@ def _link_receipt_impl(ctx): _action(ctx, "extension-receipt", {"library": ctx.file.library.path, "registry": ctx.file.registry.path, "members": members, "selectedExtensions": ctx.attr.selected_extensions, "moduleStems": _module_stems(ctx.attr.selected_extensions), "abi": ctx.attr.abi, "nm": nm[0].path, "output": output.path}, inputs + ctx.files._nm, [output], "AndroidSelectedStaticExtensionLink") - return [DefaultInfo(files = depset([output]))] + return [DefaultInfo(files = depset([output])), OutputGroupInfo(mobile_static_archives = depset(archives))] android_extension_receipt = rule(implementation = _link_receipt_impl, toolchains = BUN_SOURCE_TOOLCHAINS, attrs = dict(_RUNNER, **{ @@ -347,13 +371,16 @@ def _runtime_assets_impl(ctx): output = ctx.actions.declare_directory(ctx.label.name + "/assets") if len(ctx.files.payload) != 1 or not ctx.files.payload[0].is_directory: fail("App runtime assets need the actual declared native payload tree") - leaves, inputs = [], [] + leaves, inputs, archives = [], [], [] target = "android-" + ctx.attr.abi for dependency in ctx.attr.leaves: leaf = dependency[ExtensionFinalLeafInfo] if leaf.family != "native" or leaf.target != target or leaf.metadata["runtimeVersion"] != ctx.attr.version: fail("App selected carrier differs from the declared Android ABI or native version") members = leaf.metadata.get("members", [leaf.metadata]) + selected = [row["archive"] for member in members if member["sqlName"] in ctx.attr.selected_extensions for row in member.get("staticArchives", []) if row["target"] == target] + # Retain the actual producer Files for the configured reuse regression. + archives += [file for file in dependency[OutputGroupInfo].mobile_static_archives.to_list() if file.path in selected] leaves.append({"product": leaf.product, "owner": leaf.owner, "version": leaf.version, "target": leaf.target, "runtimeVersion": leaf.metadata["runtimeVersion"], "sqlNames": sorted([member["sqlName"] for member in members]), "github": [file.path for file in leaf.github], "frozen": leaf.frozen.path}) @@ -363,16 +390,17 @@ def _runtime_assets_impl(ctx): "icu": ctx.file.icu.path if ctx.file.icu else None, "receipt": ctx.file.receipt.path, "output": output.path}, ctx.files.payload + [ctx.file.receipt] + ctx.files.icu + inputs, [output], "ExpoSelectedRuntimeAssets", tools = [ctx.attr._formatter[DefaultInfo].files_to_run]) - return [DefaultInfo(files = depset([output]))] + return [DefaultInfo(files = depset([output])), OutputGroupInfo(mobile_raw_archives = depset(archives), mobile_linked_archives = ctx.attr.receipt[OutputGroupInfo].mobile_static_archives)] android_app_runtime_assets = rule(implementation = _runtime_assets_impl, toolchains = BUN_SOURCE_TOOLCHAINS, attrs = dict(_RUNNER, **{ - "payload": attr.label(default = Label("//src/native/runtime:public_payload"), cfg = "exec", allow_files = True), - "icu": attr.label(allow_single_file = True, cfg = "exec"), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), + "payload": attr.label(default = Label("//src/native/runtime:public_payload"), cfg = android_product_inputs, allow_files = True), + "icu": attr.label(allow_single_file = True, cfg = android_product_inputs), "version": attr.string(mandatory = True), "selected_extensions": attr.string_list(mandatory = True), "mobile_static_modules": attr.string_list(mandatory = True), "receipt": attr.label(mandatory = True, allow_single_file = [".json"]), "abi": attr.string(mandatory = True, values = ["arm64-v8a", "x86_64"]), - "leaves": attr.label_list(mandatory = True, providers = [ExtensionFinalLeafInfo], cfg = "exec"), + "leaves": attr.label_list(mandatory = True, providers = [ExtensionFinalLeafInfo], cfg = android_product_inputs), "_helpers": attr.label_list(default = _STATIC_CARRIER_HELPERS, allow_files = True), "_formatter": attr.label(default = Label("//src/native/runtime/packaging:resources"), cfg = "exec", executable = True), })) @@ -397,7 +425,7 @@ android_jni_aar = rule(implementation = _jni_aar_impl, toolchains = BUN_SOURCE_T _ANDROID_MIN_SDK = str(Label("@rules_android//rules:min_sdk_version")) def _apk_abi_impl(settings, attr): - platform = "//build_defs/android:android_" + ("arm64" if attr.abi == "arm64-v8a" else "x86_64") + platform = str(ANDROID_TARGET_PLATFORMS["android-" + attr.abi]) return {"//command_line_option:android_platforms": [platform], "//command_line_option:compilation_mode": "opt", _ANDROID_MIN_SDK: attr.min_sdk_version} diff --git a/build_defs/android/configuration.bzl b/build_defs/android/configuration.bzl new file mode 100644 index 000000000..3465a25a9 --- /dev/null +++ b/build_defs/android/configuration.bzl @@ -0,0 +1,32 @@ +"""Keep released producer inputs independent of the consuming Android app.""" + +load("@oliphaunt_toolchain_metadata//:android-platforms.bzl", "ANDROID_PLATFORMS") + +ANDROID_TARGET_PLATFORMS = {"android-arm64-v8a": Label("//build_defs/android:android_arm64"), "android-x86_64": Label("//build_defs/android:android_x86_64")} +_MIN_SDK = str(Label("@rules_android//rules:min_sdk_version")) + +def _native_inputs_impl(_settings, attr): + targets = [target for target, platform in ANDROID_TARGET_PLATFORMS.items() if platform == attr.platform] + if len(targets) != 1: + fail("Android native construction needs a declared ABI platform") + return {"//command_line_option:android_platforms": [str(attr.platform)], _MIN_SDK: ANDROID_PLATFORMS[targets[0]]["api"]} + +# Set the product floor before upstream's split; never lower an app's JVM floor. +android_native_inputs = transition(implementation = _native_inputs_impl, inputs = [], outputs = ["//command_line_option:android_platforms", _MIN_SDK]) + +def _product_inputs_impl(_settings, _attr): + # These are Linux-produced archives/data, not tools or APK dependencies. + # The producer owns any outgoing ABI split and its minimum SDK. + return { + "//command_line_option:platforms": [str(Label("//build_defs/native:linux_x86_64_glibc_2_28"))], + "//command_line_option:android_platforms": [], + "//command_line_option:Android configuration distinguisher": "main", + "//command_line_option:dynamic_mode": "default", + _MIN_SDK: 0, + } + +android_product_inputs = transition( + implementation = _product_inputs_impl, + inputs = [], + outputs = ["//command_line_option:platforms", "//command_line_option:android_platforms", "//command_line_option:Android configuration distinguisher", "//command_line_option:dynamic_mode", _MIN_SDK], +) diff --git a/build_defs/android/reuse-test.bzl b/build_defs/android/reuse-test.bzl new file mode 100644 index 000000000..019d367e8 --- /dev/null +++ b/build_defs/android/reuse-test.bzl @@ -0,0 +1,42 @@ +"""Compare the actual release and app-link archive Files for each supported ABI.""" + +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("@rules_android//rules:android_split_transition.bzl", "android_split_transition") +load(":app.bzl", "android_app_configuration") +load(":configuration.bzl", "ANDROID_TARGET_PLATFORMS") + +def _apps_impl(settings, attr): + return {target + "/" + str(minimum): android_app_configuration(settings, struct(abi = target.removeprefix("android-"), min_sdk_version = minimum)) for target in ANDROID_TARGET_PLATFORMS for minimum in [attr.min_sdk_version, attr.min_sdk_version + 1]} + +_apps = transition(implementation = _apps_impl, inputs = [], outputs = ["//command_line_option:android_platforms", "//command_line_option:compilation_mode", str(Label("@rules_android//rules:min_sdk_version"))]) + +def _view_impl(ctx): + return [ctx.attr.assets[0][OutputGroupInfo]] + +_view = rule(implementation = _view_impl, attrs = { + "assets": attr.label(default = "//src/examples/native/react-native-expo:selected_runtime_assets", cfg = android_split_transition), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), +}) + +def _reuse_impl(ctx): + env = unittest.begin(ctx) + asserts.equals(env, sorted([target + "/" + str(minimum) for target in ANDROID_TARGET_PLATFORMS for minimum in [ctx.attr.min_sdk_version, ctx.attr.min_sdk_version + 1]]), sorted(ctx.split_attr.apps)) + for abi, targets in ctx.split_attr.apps.items(): + asserts.equals(env, 1, len(targets)) + views = targets[0][OutputGroupInfo] + raw = views.mobile_raw_archives.to_list() + linked = views.mobile_linked_archives.to_list() + asserts.true(env, len(raw) > 0, abi + " must observe actual release archives") + asserts.equals(env, sorted([file.path for file in raw]), sorted([file.path for file in linked]), abi + " release and APK link must consume the same configured archive Files") + return unittest.end(env) + +_reuse_test = unittest.make(_reuse_impl, attrs = { + "apps": attr.label_list(cfg = _apps), + "min_sdk_version": attr.int(mandatory = True), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), +}) + +def android_cpp_reuse_test(name, min_sdk_version): + _view(name = name + "_view", testonly = True, tags = ["manual"]) + _reuse_test(name = name, apps = [":" + name + "_view"], min_sdk_version = min_sdk_version, + target_compatible_with = ["@platforms//os:linux", "@platforms//cpu:x86_64"], size = "small") diff --git a/build_defs/android/toolchain.bzl b/build_defs/android/toolchain.bzl index 6d25f155c..aa5dd0cd8 100644 --- a/build_defs/android/toolchain.bzl +++ b/build_defs/android/toolchain.bzl @@ -5,6 +5,8 @@ load("@rules_cc//cc:action_names.bzl", "ACTION_NAMES") load("@rules_cc//cc:cc_toolchain_config_lib.bzl", "feature", "flag_group", "flag_set") load("@rules_cc//cc/common:cc_common.bzl", "cc_common") load("@rules_cc//cc/toolchains:cc_toolchain_config_info.bzl", "CcToolchainConfigInfo") +load("@rules_android//rules:android_split_transition.bzl", "android_split_transition") +load(":configuration.bzl", "android_native_inputs") def _android_cc_config_impl(ctx): # This is a qualification bound, not a second version authority. An SDK or @@ -70,11 +72,6 @@ android_cc_config = rule( provides = [CcToolchainConfigInfo], ) -def _platform_impl(settings, attr): - return {"//command_line_option:platforms": [str(attr.platform)]} - -_android_platform = transition(implementation = _platform_impl, inputs = [], outputs = ["//command_line_option:platforms"]) - def _android_artifact_impl(ctx): files = ctx.attr.artifact[0][DefaultInfo].files if ctx.attr.output: @@ -88,8 +85,9 @@ def _android_artifact_impl(ctx): android_artifact = rule( implementation = _android_artifact_impl, + cfg = android_native_inputs, attrs = { - "artifact": attr.label(mandatory = True, cfg = _android_platform), + "artifact": attr.label(mandatory = True, cfg = android_split_transition), "platform": attr.label(mandatory = True), "output": attr.string(default = ""), "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), diff --git a/build_defs/extensions/BUILD.bazel b/build_defs/extensions/BUILD.bazel index 7160ffc69..12f695aa8 100644 --- a/build_defs/extensions/BUILD.bazel +++ b/build_defs/extensions/BUILD.bazel @@ -17,12 +17,19 @@ load(":family_carriers.bzl", "native_extension_final_families") load(":forced_header_inputs_test.bzl", "native_forced_header_input_tests") load(":windows_module_test.bzl", "native_windows_module_tests") load(":mobile_host_profile_test.bzl", "mobile_host_profile_tests") +load(":mobile_cpp_reuse_test.bzl", "mobile_cpp_reuse_test") package(default_visibility = ["//visibility:public"]) native_forced_header_input_tests() native_windows_module_tests() mobile_host_profile_tests() +mobile_cpp_reuse_test( + name = "mobile_cpp_reuse_test", + target_under_test = "//build_defs/extensions/mobile/final:oliphaunt_extension_pg_hashids_ios_xcframework", + target_compatible_with = ["@platforms//os:macos", "@platforms//cpu:aarch64"], + size = "small", +) exports_files(["contrib.bzl", "extensions.bzl", "repositories.bzl", "uuid_config.h", "pgtap_upstream_parity_test.mts", "validate_member_payload.mts"]) exports_files(["native_extension_all_sql_test.sh"]) diff --git a/build_defs/extensions/mobile_carriers.bzl b/build_defs/extensions/mobile_carriers.bzl index 63bb03e6e..8e190d7d4 100644 --- a/build_defs/extensions/mobile_carriers.bzl +++ b/build_defs/extensions/mobile_carriers.bzl @@ -2,7 +2,10 @@ load("@oliphaunt_metadata//:metadata.bzl", "EXTENSIONS", "EXTENSION_GITHUB_ARTIFACTS", "EXTENSION_RELEASE_METADATA", "NATIVE_COMPONENT_SOURCES", "NATIVE_MOBILE_COMPONENT_CLOSURES", "PUBLICATION_CARRIERS") load("@oliphaunt_toolchain_metadata//:apple-platforms.bzl", "APPLE_DEPLOYMENT_TARGETS") +load("@rules_android//rules:android_split_transition.bzl", "android_split_transition") +load("//build_defs/android:configuration.bzl", "ANDROID_TARGET_PLATFORMS", "android_native_inputs") load("@rules_apple//apple:apple.bzl", "apple_static_xcframework") +load("@rules_apple//apple/internal:transition_support.bzl", "transition_support") load("//build_defs/apple:slices.bzl", "apple_slice_filegroup", "apple_slice_groups") load("//build_defs/foundation:bun.bzl", "BUN_SOURCE_TOOLCHAINS", "bun_file_action") load("//build_defs/release:shipping.bzl", "shipping_export", "shipping_package") @@ -13,16 +16,27 @@ load(":mobile.bzl", "MobileComponentInfo", "MobileExtensionInfo") load(":mobile_members.bzl", "MOBILE_COMPONENT_ARCHIVE_OBJECTS", "MOBILE_COMPONENT_TARGETS", "MOBILE_MEMBER_ARCHIVE_OBJECTS", "MOBILE_MEMBER_TARGETS") load(":owner_carriers.bzl", "native_extension_final_contract_test", "native_final_leaf_files", "native_final_leaf_input", "native_owner_carrier_helpers", "native_owner_carrier_legal", "native_owner_carrier_metadata") -_TARGET_PLATFORMS = {"android-arm64-v8a": "//build_defs/android:android_arm64", "android-x86_64": "//build_defs/android:android_x86_64", "macos-arm64": "//build_defs/host:apple_package_macos_arm64", "ios-device": "//build_defs/host:ios_device_arm64", "ios-simulator": "//build_defs/host:ios_simulator_arm64"} +_APPLE_COHORTS = { + transition_support.xcframework_split_attr_key(arch = "arm64", environment = environment, minimum_os_version = APPLE_DEPLOYMENT_TARGETS["package_macos" if platform == "macos" else "ios"], platform_type = platform): target + for platform, environment, target in [("macos", "device", "macos-arm64"), ("ios", "device", "ios-device"), ("ios", "simulator", "ios-simulator")] +} _TARGETS = ["android-arm64-v8a", "android-x86_64", "ios-xcframework"] _MAC = ["@platforms//os:macos", "@platforms//cpu:aarch64"] _LINUX = ["@platforms//os:linux", "@platforms//cpu:x86_64"] -def _slices_impl(settings, attr): - targets = ["macos-arm64", "ios-device", "ios-simulator"] if attr.target == "ios-xcframework" else [attr.target] - return {target: {"//command_line_option:platforms": [str(Label(_TARGET_PLATFORMS[target]))]} for target in targets} +_AndroidSlicesInfo = provider(fields = ["members", "dependencies"]) -_slices = transition(implementation = _slices_impl, inputs = [], outputs = ["//command_line_option:platforms"]) +def _android_slices_impl(ctx): + return [_AndroidSlicesInfo(members = ctx.attr.members, dependencies = ctx.attr.dependencies)] + +# Scope Android's upstream split to the compiled slices, keeping Linux host +# archives in their original configuration and sharing objects with APK deps. +_android_slices = rule(implementation = _android_slices_impl, cfg = android_native_inputs, attrs = { + "platform": attr.label(mandatory = True), + "members": attr.label_list(providers = [MobileExtensionInfo], cfg = android_split_transition), + "dependencies": attr.label_list(providers = [MobileComponentInfo], cfg = android_split_transition), + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), +}) def _host_impl(settings, attr): # Reused desktop dylibs retain the direct carrier contract, unlike XCFramework slices. @@ -37,6 +51,12 @@ def _apple_package_impl(settings, attr): _apple_package = transition(implementation = _apple_package_impl, inputs = [], outputs = [_MACOS_PROFILE]) +# The stock slice transition must not duplicate the existing desktop producers. +def _ios_carrier_host_impl(settings, attr): + return _host_impl(settings, attr) | {_MACOS_PROFILE: str(Label("@apple_support//platforms:macos_arm64"))} + +_ios_carrier_host = transition(implementation = _ios_carrier_host_impl, inputs = [], outputs = ["//command_line_option:platforms", _MACOS_PROFILE]) + def _framework_impl(ctx): archive = ctx.attr.archive[0] return [archive[DefaultInfo], OutputGroupInfo(**apple_slice_groups([archive]))] @@ -58,13 +78,20 @@ def _carriers_impl(ctx): if sorted(host.keys()) != expected or len(host) != len(host_members): fail("mobile final leaf must consume every canonical host member exactly once") compiled = sorted([member["sqlName"] for member in metadata["members"] if member["descriptor"].get("native-module-file")]) - statics = {target: {member[MobileExtensionInfo].sql_name: member[MobileExtensionInfo] for member in members} for target, members in ctx.split_attr.members.items()} - dependencies = {target: {item[MobileComponentInfo].identity: item[MobileComponentInfo] for item in rows} for target, rows in ctx.split_attr.dependencies.items()} + if ctx.attr.target == "ios-xcframework": + member_slices = {_APPLE_COHORTS[target]: rows for target, rows in ctx.split_attr.members.items()} + dependency_slices = {_APPLE_COHORTS[target]: rows for target, rows in ctx.split_attr.dependencies.items()} + else: + slices = ctx.attr.slices[_AndroidSlicesInfo] + member_slices = {ctx.attr.target: slices.members} + dependency_slices = {ctx.attr.target: slices.dependencies} + statics = {target: {member[MobileExtensionInfo].sql_name: member[MobileExtensionInfo] for member in members} for target, members in member_slices.items()} + dependencies = {target: {item[MobileComponentInfo].identity: item[MobileComponentInfo] for item in rows} for target, rows in dependency_slices.items()} inputs = [ctx.file._components] + ctx.files.legal slice_files = {target: [] for target in (statics | dependencies)} if ctx.attr.target == "ios-xcframework" else {} tools = [] for target, rows in statics.items(): - if sorted(rows.keys()) != compiled or len(rows) != len(ctx.split_attr.members[target]): + if sorted(rows.keys()) != compiled or len(rows) != len(member_slices[target]): fail("mobile static slice differs from canonical compiled member cohort") for member in rows.values(): inputs += [member.archive, member.registration] @@ -73,7 +100,7 @@ def _carriers_impl(ctx): tools.append(member.tools) dependency_ids = sorted({identity: True for name in expected for identity in NATIVE_MOBILE_COMPONENT_CLOSURES[name][ctx.attr.target]["linkUnits"]}.keys()) for target, rows in dependencies.items(): - if sorted(rows.keys()) != dependency_ids or len(rows) != len(ctx.split_attr.dependencies[target]): + if sorted(rows.keys()) != dependency_ids or len(rows) != len(dependency_slices[target]): fail("mobile component slice differs from canonical member closure") for component in rows.values(): inputs.append(component.archive) @@ -121,12 +148,21 @@ def _carriers_impl(ctx): slice_groups[name] = depset(files, transitive = [slice_groups[name]] if name in slice_groups else []) return [DefaultInfo(files = depset([github, frozen, manifest])), ExtensionFinalLeafInfo(product = metadata["product"], owner = metadata["owner"], version = metadata["version"], family = "native", target = ctx.attr.target, metadata = spec, github = [github], npm = [], cargo = [], manifest = manifest, frozen = frozen, spec = specification, validation = None, receipt = None, origin = None), - OutputGroupInfo(github = depset([github]), manifest = depset([manifest]), frozen = depset([frozen]), spec = depset([specification]), **slice_groups)] + OutputGroupInfo(github = depset([github]), manifest = depset([manifest]), frozen = depset([frozen]), spec = depset([specification]), mobile_static_archives = depset([member.archive for rows in statics.values() for member in rows.values()]), **slice_groups)] + +def _carrier_attributes(slices = None, host = mobile_host_transition): + return { + "product": attr.string(mandatory = True), "target": attr.string(mandatory = True, values = _TARGETS), "host": attr.label_list(mandatory = True, providers = [ExtensionMemberInfo], cfg = host), + "frameworks": attr.label_keyed_string_dict(allow_files = True), "legal": attr.label_list(allow_files = True), + "_tool": attr.label(default = "//build_defs/extensions:mobile_carriers.mts", allow_single_file = True), "_helpers": attr.label_list(default = mobile_carrier_helpers(), allow_files = True), "_components": attr.label(default = "//src/extensions/catalog:native-components.toml", allow_single_file = True), "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), + } | ({"members": attr.label_list(providers = [MobileExtensionInfo], cfg = slices), "dependencies": attr.label_list(providers = [MobileComponentInfo], cfg = slices)} if slices else {"slices": attr.label(providers = [_AndroidSlicesInfo], mandatory = True)}) -_mobile_carriers = rule(implementation = _carriers_impl, toolchains = BUN_SOURCE_TOOLCHAINS, attrs = { - "product": attr.string(mandatory = True), "target": attr.string(mandatory = True, values = _TARGETS), "host": attr.label_list(mandatory = True, providers = [ExtensionMemberInfo], cfg = mobile_host_transition), - "members": attr.label_list(providers = [MobileExtensionInfo], cfg = _slices), "dependencies": attr.label_list(providers = [MobileComponentInfo], cfg = _slices), "frameworks": attr.label_keyed_string_dict(allow_files = True), "legal": attr.label_list(allow_files = True), - "_tool": attr.label(default = "//build_defs/extensions:mobile_carriers.mts", allow_single_file = True), "_helpers": attr.label_list(default = mobile_carrier_helpers(), allow_files = True), "_components": attr.label(default = "//src/extensions/catalog:native-components.toml", allow_single_file = True), "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), +_mobile_carriers = rule(implementation = _carriers_impl, toolchains = BUN_SOURCE_TOOLCHAINS, attrs = _carrier_attributes()) +# Raw inventories and XCFrameworks consume the same configured object Files. +_mobile_ios_carriers = rule(implementation = _carriers_impl, toolchains = BUN_SOURCE_TOOLCHAINS, cfg = _apple_package, attrs = _carrier_attributes(transition_support.xcframework_transition, host = _ios_carrier_host) | { + "ios": attr.string_list_dict(default = {"device": ["arm64"], "simulator": ["arm64"]}), + "macos": attr.string_list(default = ["arm64"]), + "minimum_os_versions": attr.string_dict(default = {"macos": APPLE_DEPLOYMENT_TARGETS["package_macos"], "ios": APPLE_DEPLOYMENT_TARGETS["ios"]}), }) def mobile_native_carriers(name, product, target, frameworks = {}): @@ -135,8 +171,13 @@ def mobile_native_carriers(name, product, target, frameworks = {}): hosts = {row.name: row.label for row in desktop_member_inventory(include_postgis = True)} units = sorted({identity: True for sql_name in names for identity in NATIVE_MOBILE_COMPONENT_CLOSURES[sql_name][target]["linkUnits"]}.keys()) compatible = _MAC if target == "ios-xcframework" else _LINUX - _mobile_carriers(name = name, product = product, target = target, host = [hosts[sql_name] for sql_name in names], members = [MOBILE_MEMBER_TARGETS[sql_name] for sql_name in names if sql_name in MOBILE_MEMBER_TARGETS], - dependencies = [MOBILE_COMPONENT_TARGETS[identity] for identity in units], frameworks = frameworks, legal = native_owner_carrier_legal(product), target_compatible_with = compatible) + slices = {"members": [MOBILE_MEMBER_TARGETS[sql_name] for sql_name in names if sql_name in MOBILE_MEMBER_TARGETS], "dependencies": [MOBILE_COMPONENT_TARGETS[identity] for identity in units]} + if target != "ios-xcframework": + _android_slices(name = name + "_slices", platform = ANDROID_TARGET_PLATFORMS[target], **slices) + slices = {"slices": ":" + name + "_slices"} + carrier = _mobile_ios_carriers if target == "ios-xcframework" else _mobile_carriers + carrier(name = name, product = product, target = target, host = [hosts[sql_name] for sql_name in names], + frameworks = frameworks, legal = native_owner_carrier_legal(product), target_compatible_with = compatible, **slices) owner = extension_identity(names[0]) packages = [] github_files = name + "_github_files" @@ -235,7 +276,7 @@ def mobile_final_carrier_targets(): for target in _TARGETS: suffix = target.replace("-", "_") compatible = _MAC if target == "ios-xcframework" else _LINUX - shipping_export(name = suffix + "_shipping_export", packages = packages[target], qualification = [":" + suffix + "_final_contract_test"], testonly = True, target_compatible_with = compatible) + shipping_export(name = suffix + "_shipping_export", packages = packages[target], qualification = [":" + suffix + "_final_contract_test"] + (["//build_defs/extensions:mobile_cpp_reuse_test"] if target == "ios-xcframework" else []), testonly = True, target_compatible_with = compatible) native_extension_final_contract_test(name = suffix + "_final_contract_test", products = products, leaves = [":" + product.replace("-", "_") + "_" + suffix for product in products], target_compatible_with = compatible, size = "medium") selected = sorted({row["artifactProduct"]: True for row in EXTENSION_RELEASE_METADATA if row["sqlName"] in ["pg_uuidv7", "pgtap"]}.keys()) diff --git a/build_defs/extensions/mobile_cpp_reuse_test.bzl b/build_defs/extensions/mobile_cpp_reuse_test.bzl new file mode 100644 index 000000000..f018eadc6 --- /dev/null +++ b/build_defs/extensions/mobile_cpp_reuse_test.bzl @@ -0,0 +1,54 @@ +"""Require raw and framework inventories to share their actual compiled objects.""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load(":mobile_members.bzl", "MOBILE_MEMBER_ARCHIVE_OBJECTS") + +_ObjectsInfo = provider(fields = ["profiles", "branches"]) +_MEMBER = Label(MOBILE_MEMBER_ARCHIVE_OBJECTS["pg_hashids"]) +_EDGES = ["members", "dependencies", "frameworks", "archive", "deps", "roots", "actual", "library"] + +def _objects_impl(target, ctx): + profiles = {} + branches = {} + if target.label == _MEMBER: + profile = "macos-arm64" if ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]) else "ios-device" if ctx.target_platform_has_constraint(ctx.attr._device[platform_common.ConstraintValueInfo]) else "ios-simulator" + objects = [file.path for item in target[CcInfo].linking_context.linker_inputs.to_list() for library in item.libraries for group in [library.objects, library.pic_objects] for file in (group or [])] + if not objects: + fail("The reuse probe needs the member's actual compiled object Files") + profiles[profile] = [depset(["\n".join(sorted(objects))])] + if ctx.rule != None: + for edge in _EDGES: + branch = {} + dependencies = getattr(ctx.rule.attr, edge, []) + if type(dependencies) == "dict": + dependencies = dependencies.keys() + elif type(dependencies) != "list": + dependencies = [dependencies] + for dependency in dependencies: + if type(dependency) == "Target" and _ObjectsInfo in dependency: + for profile, files in dependency[_ObjectsInfo].profiles.items(): + profiles.setdefault(profile, []).append(files) + branch.setdefault(profile, []).append(files) + if edge in ["members", "frameworks"]: + branches[edge] = {profile: depset(transitive = files) for profile, files in branch.items()} + return [_ObjectsInfo(profiles = {profile: depset(transitive = files) for profile, files in profiles.items()}, branches = branches)] + +_objects = aspect(implementation = _objects_impl, attr_aspects = _EDGES, attrs = { + "_macos": attr.label(default = "@platforms//os:macos"), + "_device": attr.label(default = "@apple_support//constraints:device"), +}) + +def _reuse_test(ctx): + env = analysistest.begin(ctx) + branches = analysistest.target_under_test(env)[_ObjectsInfo].branches + for branch in ["members", "frameworks"]: + profiles = branches.get(branch, {}) + asserts.equals(env, ["ios-device", "ios-simulator", "macos-arm64"], sorted(profiles), branch + " must expose every compiled slice") + for profile, files in profiles.items(): + asserts.equals(env, 1, len(files.to_list()), branch + "/" + profile + " must expose one compiled object set") + for profile, files in branches.get("members", {}).items(): + asserts.equals(env, files.to_list(), branches.get("frameworks", {}).get(profile, depset()).to_list(), profile + " raw inventories and frameworks must reuse the same compiled object Files") + return analysistest.end(env) + +mobile_cpp_reuse_test = analysistest.make(_reuse_test, extra_target_under_test_aspects = [_objects]) diff --git a/build_defs/extensions/native_carriers.mts b/build_defs/extensions/native_carriers.mts index 966a160e9..ec887968c 100644 --- a/build_defs/extensions/native_carriers.mts +++ b/build_defs/extensions/native_carriers.mts @@ -24,6 +24,7 @@ import { stageExtensionNativeNpmPackages, } from '../../src/extensions/artifacts/packages/tools/package-extension-release-carriers.mts'; import { + COMPILER_RT_NATIVE_CONSTRUCTION_MEMBERS, extensionCarrierLegalContract, nativeExtensionConstructionLicenseInventory, stageExtensionUpstreamLicenses, @@ -128,7 +129,8 @@ export function frozenNativeMember(spec: NativeCarrierSpec, actual: Record 0)) || (spec.contrib && diff --git a/build_defs/foundation/repositories.bzl b/build_defs/foundation/repositories.bzl index 3a8de58ef..a3fd09416 100644 --- a/build_defs/foundation/repositories.bzl +++ b/build_defs/foundation/repositories.bzl @@ -173,6 +173,7 @@ def _toolchain_metadata_impl(ctx): fail("Cannot import pinned language tools: " + result.stderr) data = json.decode(result.stdout) ctx.file("apple-platforms.bzl", "APPLE_DEPLOYMENT_TARGETS = " + repr(data["apple_deployment_targets"]) + "\n") + ctx.file("android-platforms.bzl", "ANDROID_PLATFORMS = " + repr(data["android_platforms"]) + "\n") ctx.file("toolchains.json", json.encode(data["language_acquisition"])) ctx.file("node-version", data["node"] + "\n") ctx.file("typescript.json", json.encode({ @@ -183,7 +184,7 @@ def _toolchain_metadata_impl(ctx): ctx.file("query-package-type.bzl", "QUERY_PACKAGE_TYPE = " + repr(data["query_package_type"]) + "\nQUERY_NPM_NAME = " + repr(data["query_npm_name"]) + "\n") rust = data["query_rust"] ctx.file("query-rust.bzl", "QUERY_RUST_CRATE_NAME = " + repr(rust["crate_name"]) + "\nQUERY_RUST_EDITION = " + repr(rust["edition"]) + "\nQUERY_RUST_VERSION = " + repr(rust["version"]) + "\nQUERY_RUST_ENV = " + repr(rust["env"]) + "\n") - ctx.file("BUILD.bazel", 'exports_files(["toolchains.json", "node-version", "typescript.json", "query-package-type.bzl", "query-rust.bzl", "apple-platforms.bzl"], visibility = ["//visibility:public"])\n') + ctx.file("BUILD.bazel", 'exports_files(["toolchains.json", "node-version", "typescript.json", "query-package-type.bzl", "query-rust.bzl", "apple-platforms.bzl", "android-platforms.bzl"], visibility = ["//visibility:public"])\n') return ctx.repo_metadata(reproducible = True) toolchain_metadata_repository = repository_rule( diff --git a/build_defs/foundation/toolchain-metadata.mts b/build_defs/foundation/toolchain-metadata.mts index c3ea1f1d6..46fb0679c 100644 --- a/build_defs/foundation/toolchain-metadata.mts +++ b/build_defs/foundation/toolchain-metadata.mts @@ -1,5 +1,8 @@ import { readFileSync } from 'node:fs'; -import { platformCompatibilityContract } from '../../tools/release/platform-compatibility-policy.mts'; +import { + platformCompatibilityContract, + platformCompatibilityTargets, +} from '../../tools/release/platform-compatibility-policy.mts'; import { appleDeploymentTargets } from '../host/deployment-policy.mts'; export function mobileRustTargets(): string[] { @@ -159,6 +162,14 @@ export function importToolchainMetadata( }; return { apple_deployment_targets: appleDeploymentTargets(), + android_platforms: Object.fromEntries( + platformCompatibilityTargets() + .filter((target) => target.startsWith('android-')) + .map((target) => [ + target, + { api: platformCompatibilityContract(target).elf.androidApiLevel }, + ]), + ), node: versions.node, bun: versions.bun, typescript: projection(typescript), diff --git a/build_defs/foundation/toolchain-metadata.test.mts b/build_defs/foundation/toolchain-metadata.test.mts index f746b4dae..27946d25c 100644 --- a/build_defs/foundation/toolchain-metadata.test.mts +++ b/build_defs/foundation/toolchain-metadata.test.mts @@ -67,6 +67,13 @@ test('Apple construction floors project the public compatibility policy without }); }); +test('Android slice floors project the public compatibility policy without NDK acquisition', () => { + expect(importToolchainMetadata(...fixture()).android_platforms).toEqual({ + 'android-arm64-v8a': { api: 24 }, + 'android-x86_64': { api: 24 }, + }); +}); + test.each([ 'Cargo version', 'npm type', diff --git a/build_defs/host/BUILD.bazel b/build_defs/host/BUILD.bazel index f12962f44..85219b1a8 100644 --- a/build_defs/host/BUILD.bazel +++ b/build_defs/host/BUILD.bazel @@ -4,7 +4,7 @@ load("@rules_cc//cc:cc_test.bzl", "cc_test") load("@rules_shell//shell:sh_test.bzl", "sh_test") load("@rules_swift//swift:swift.bzl", "swift_binary") load("@rules_swift//swift/toolchains:xcode_swift_toolchain.bzl", "xcode_swift_toolchain") -load(":profile-test.bzl", "profile_test") +load(":profile-test.bzl", "apple_sdk_inputs_test", "profile_test") load(":cc-profiles.bzl", "host_cc_profiles") package(default_visibility = ["//visibility:public"]) @@ -25,6 +25,12 @@ bun_test(name = "host_sdk_test", entry_point = "host-sdk.test.mts", data = ["hos bun_test(name = "apple_app_tools_test", entry_point = "apple-app-tools.test.mts", data = ["apple-app-tools.mts"], size = "small") bun_binary(name = "sdk_tree_conformance", entry_point = "sdk-tree-conformance.mts", data = ["sdk-tree.bzl"], tags = ["manual"]) profile_test(name = "profile_analysis_test", size = "small") +apple_sdk_inputs_test( + name = "apple_sdk_inputs_test", + target_under_test = ":host_c_smoke", + target_compatible_with = ["@platforms//os:macos", "@platforms//cpu:aarch64"], + size = "small", +) xcode_swift_toolchain( name = "xcode_swift_base", diff --git a/build_defs/host/cc-profiles.bzl b/build_defs/host/cc-profiles.bzl index 9a5732937..b5ddc0d4d 100644 --- a/build_defs/host/cc-profiles.bzl +++ b/build_defs/host/cc-profiles.bzl @@ -86,7 +86,7 @@ def host_cc_profiles(sdk): supports_parse_showincludes = True, shorten_virtual_includes = True, ) - _compiler("cc_compiler", ":configuration", sdk, []) + _compiler("cc_compiler", ":configuration", sdk) else: for name, minimum in [("native", sdk["deploymentTargets"]["native_macos"])]: unix_config( @@ -126,7 +126,7 @@ def host_cc_profiles(sdk): ios_floor = sdk["deploymentTargets"]["ios"], use_ar = True, ) - _compiler("cc_" + name, ":configuration_" + name, sdk, [component for other in ["iphoneos", "iphonesimulator"] for component in ["sdk_" + other, "platform_" + other + "/Library", "platform_" + other + "/usr/lib"]]) + _compiler("cc_" + name, ":configuration_" + name, sdk) # Build the pinned standard Apple drivers with the declared native # compiler. Upstream's universal_exec_tool uses an ambient shell compiler. cc_binary(name = "wrapped_clang", srcs = ["wrapped_clang.cc"], copts = ["-std=c++17"]) @@ -165,10 +165,12 @@ def host_cc_profiles(sdk): macos_floor = sdk["deploymentTargets"]["package_macos"], ios_floor = sdk["deploymentTargets"]["ios"], ) - _compiler("cc_" + name, ":configuration_" + name, sdk, [component for other in ["macosx", "iphoneos", "iphonesimulator"] if other != selected_sdk for component in ["sdk_" + other, "platform_" + other + "/Library", "platform_" + other + "/usr/lib"]], drivers = True) + _compiler("cc_" + name, ":configuration_" + name, sdk, drivers = True) -def _compiler(name, configuration, sdk, exclude, drivers = False): - components = [component for component in sdk["components"] if component["name"] not in exclude] +def _compiler(name, configuration, sdk, drivers = False): + # A shared SDK closure lets sandbox reuse retain its symlinks across Apple + # profiles instead of deleting and recreating thousands on every switch. + components = sdk["components"] for role in ["compile", "link"]: native.filegroup( name = name + "_" + role + "_files", diff --git a/build_defs/host/profile-test.bzl b/build_defs/host/profile-test.bzl index 82f5b02ee..76c77f40c 100644 --- a/build_defs/host/profile-test.bzl +++ b/build_defs/host/profile-test.bzl @@ -209,6 +209,26 @@ def _profile_test(ctx): _profile_rule_test = analysistest.make(_profile_test, fragments = ["cpp"], attrs = {"source": attr.label(default = ":sdk-smoke.c", allow_single_file = True), "libtool": attr.label(default = "@apple_support//crosstool:libtool.cc", allow_single_file = True), "cc_profiles": attr.label_list(providers = [cc_common.CcToolchainInfo]), "archive_profiles": attr.label_list(providers = [cc_common.CcToolchainInfo], cfg = _archive_fixture), "xcode_configuration": attr.label(providers = [apple_common.XcodeVersionConfig]), "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist")}) +def _apple_sdk_inputs_test(ctx): + env = analysistest.begin(ctx) + for role, profiles in [("compile", ctx.attr.compiler_inputs), ("link", ctx.attr.linker_inputs)]: + expected = None + for profile in profiles: + paths = sorted([source.short_path for source in profile[DefaultInfo].files.to_list() if "/sdk_" in source.short_path or "/platform_" in source.short_path]) + asserts.true(env, bool(paths), "%s has no SDK inputs" % profile.label) + if expected == None: + expected = paths + asserts.true(env, paths == expected, "%s SDK inputs differ for %s; switching profiles would rebuild the sandbox SDK tree" % (role, profile.label)) + return analysistest.end(env) + +apple_sdk_inputs_test = analysistest.make( + _apple_sdk_inputs_test, + attrs = { + "compiler_inputs": attr.label_list(default = [Label("@native_apple_sdk//:cc_" + profile + "_compile_files") for profile in ["native", "package", "ios", "ios_simulator"]]), + "linker_inputs": attr.label_list(default = [Label("@native_apple_sdk//:cc_" + profile + "_link_files") for profile in ["native", "package", "ios", "ios_simulator"]]), + }, +) + def profile_test(name, **kwargs): """Exercise builtin path interpretation without acquiring a licensed SDK.""" xcode_version(name = name + "_xcode_version", version = "26.5.0.17f42", default_macos_sdk_version = "26.5", default_ios_sdk_version = "26.5") diff --git a/build_defs/native/android/runtime.bzl b/build_defs/native/android/runtime.bzl index 56eca406e..6c95cf6a9 100644 --- a/build_defs/native/android/runtime.bzl +++ b/build_defs/native/android/runtime.bzl @@ -122,4 +122,4 @@ def native_android_runtime_targets(): packages.append(":" + name + "_shipping") _assets_test(name = "android_runtime_assets_test", assets = ":android_release_assets", args = bun_test_entrypoint_args("android_runtime_assets_test"), target_compatible_with = _LINUX, size = "small") - shipping_export(name = "android_shipping_export", packages = packages, qualification = [":android_runtime_assets_test", "//src/examples/native/react-native-expo:android_installed_test"], testonly = True, target_compatible_with = _LINUX) + shipping_export(name = "android_shipping_export", packages = packages, qualification = [":android_runtime_assets_test", "//src/examples/native/react-native-expo:android_installed_test", "//src/examples/native/react-native-expo:android_cpp_reuse_test"], testonly = True, target_compatible_with = _LINUX) diff --git a/src/examples/native/react-native-expo/BUILD.bazel b/src/examples/native/react-native-expo/BUILD.bazel index 94c0a1649..60ee6d7b5 100644 --- a/src/examples/native/react-native-expo/BUILD.bazel +++ b/src/examples/native/react-native-expo/BUILD.bazel @@ -18,8 +18,10 @@ load("@oliphaunt_metadata//:metadata.bzl", "COMPLETED_SUPPLIER_ARTIFACTS", "EXTE load("@rules_android//android:rules.bzl", "aar_import", "android_binary", "android_library") load("@rules_kotlin//kotlin:android.bzl", "kt_android_library") load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_import", "cc_library") -load("//build_defs/android:app.bzl", "android_aar_native", "android_apk_proof", "android_app_runtime_assets", "android_extension_receipt", "android_final_aar", "android_header_tree", "android_jni_aar", "android_workspace_npm", "expo_android_bundle", "expo_android_project", "expo_autolinking_test", "mobile_query_input", "react_native_android_codegen") +load("//build_defs/android:app.bzl", "android_aar_native", "android_apk_proof", "android_app_runtime_assets", "android_extension_receipt", "android_final_aar", "android_header_tree", "android_jni_aar", "android_native_member", "android_workspace_npm", "expo_android_bundle", "expo_android_project", "expo_autolinking_test", "mobile_query_input", "react_native_android_codegen") +load("//build_defs/android:configuration.bzl", "ANDROID_TARGET_PLATFORMS") load("//build_defs/android:native-cpp.bzl", "REACT_NATIVE_RELEASE_DEFINES") +load("//build_defs/android:reuse-test.bzl", "android_cpp_reuse_test") load("//build_defs/extensions:mobile.bzl", "mobile_extension_registry") load("//build_defs/extensions:mobile_members.bzl", "MOBILE_MEMBER_LIBRARIES", "MOBILE_MEMBER_TARGETS", "MOBILE_SQL_ONLY_MEMBERS") load("//build_defs/quality:sources.bzl", "source_quality_files") @@ -124,6 +126,7 @@ _SHARED = ["-static_link_cpp_runtimes"] _RN_NATIVE = ["@expo_android_sources//:react_native_prefab", "@expo_android_sources//:fbjni", "@expo_android_sources//:shared_cxx_runtime"] _JNA = ANDROID_NATIVE["reactNativeSdk"]["jna"] _MIN_SDK = ANDROID_NATIVE["minSdkVersion"] +android_cpp_reuse_test(name = "android_cpp_reuse_test", min_sdk_version = _MIN_SDK) android_final_aar(name = "kotlin_final_aar", source = "//src/native/sdks/kotlin:android_maven_archive") aar_import(name = "kotlin_installed", aar = ":kotlin_final_aar", package = "dev.oliphaunt", deps = [ @@ -157,10 +160,12 @@ cc_binary(name = "liboliphauntreactnative.so", deps = [":oliphaunt_reactnative_s # Static registration consumes the actual ABI-specific archive; final proof # binds its definitions to the same cohort embedded in the Hermes bundle. _DESCRIPTORS = {row["sql-name"]: row for row in EXTENSIONS.values()} -_MEMBERS = [MOBILE_MEMBER_TARGETS[name] for name in ANDROID_COMPILED_EXTENSIONS] +[android_native_member(name = "selected_" + name, member = MOBILE_MEMBER_TARGETS[name], library = MOBILE_MEMBER_LIBRARIES[name], + platform = select({_ARM64: ANDROID_TARGET_PLATFORMS["android-arm64-v8a"], _X64: ANDROID_TARGET_PLATFORMS["android-x86_64"]})) for name in ANDROID_COMPILED_EXTENSIONS] +_MEMBERS = [":selected_" + name for name in ANDROID_COMPILED_EXTENSIONS] mobile_extension_registry(name = "selected_registry", members = _MEMBERS, sql_names = ANDROID_COMPILED_EXTENSIONS) cc_binary(name = "liboliphaunt_extensions.so", srcs = [":selected_registry"], - deps = [":runtime_cc", ":runtime_headers", "@expo_android_sources//:shared_cxx_runtime"] + [MOBILE_MEMBER_LIBRARIES[name] for name in ANDROID_COMPILED_EXTENSIONS], + deps = [":runtime_cc", ":runtime_headers", "@expo_android_sources//:shared_cxx_runtime"] + _MEMBERS, features = _SHARED, linkshared = True, linkstatic = True, linkopts = ["-Wl,-z,defs", "-Wl,-z,max-page-size=16384", "-Wl,-z,common-page-size=16384", "-Wl,-soname,liboliphaunt_extensions.so"], target_compatible_with = ["@platforms//os:android"]) android_extension_receipt(name = "extension_receipt", library = ":liboliphaunt_extensions.so", registry = ":selected_registry", members = _MEMBERS, selected_extensions = ANDROID_SELECTED_EXTENSIONS, abi = _ABI) diff --git a/src/extensions/tools/extension-upstream-licenses.mts b/src/extensions/tools/extension-upstream-licenses.mts index f672d033e..7a7750294 100644 --- a/src/extensions/tools/extension-upstream-licenses.mts +++ b/src/extensions/tools/extension-upstream-licenses.mts @@ -55,6 +55,7 @@ export const PGCRYPTO_NATIVE_CONSTRUCTION_TARGETS = Object.freeze([ 'macos-arm64', 'windows-x64-msvc', ]); +export const COMPILER_RT_NATIVE_CONSTRUCTION_MEMBERS = Object.freeze(['vector', 'pg_textsearch']); const OPENSSL_EMBEDDED_NATIVE_TARGETS = new Set([ 'android-arm64-v8a', 'android-x86_64', @@ -751,7 +752,7 @@ export function nativeExtensionConstructionLicenseInventory(sqlName, target) { ].map(Object.freeze), ); } - if (!['vector', 'pg_textsearch'].includes(sqlName) || target !== 'linux-x64-gnu') { + if (!COMPILER_RT_NATIVE_CONSTRUCTION_MEMBERS.includes(sqlName) || target !== 'linux-x64-gnu') { fail('native construction legal closure has no declared source for this member/target'); } const source = JSON.parse( diff --git a/src/native/runtime/BUILD.bazel b/src/native/runtime/BUILD.bazel index d7ebd5ca2..614d49207 100644 --- a/src/native/runtime/BUILD.bazel +++ b/src/native/runtime/BUILD.bazel @@ -97,6 +97,7 @@ alias(name = "runtime_shared", actual = select({"//build_defs/native:macos_arm64 native_android_runtime_targets() native_ios_runtime_targets() native_apple_runtime_targets(qualification = [ + "//build_defs/host:apple_sdk_inputs_test", "//build_defs/native:ios_device_builtin_link_test", "//build_defs/native:ios_simulator_builtin_link_test", ":apple_final_contract_test", diff --git a/tools/bazel/README.md b/tools/bazel/README.md index 0a904c77b..d54fe8640 100644 --- a/tools/bazel/README.md +++ b/tools/bazel/README.md @@ -45,6 +45,56 @@ extension modules. Linux ARM64 and Android have separate configured targets. Cross-compilation proves construction and binary contracts; runtime behavior still requires the actual host or device. +Apple C/C++ profiles share the declared macOS, device and simulator SDK inputs. +This lets sandbox reuse retain the SDK symlinks when a build switches profiles; +the compiler flags still select each profile's own SDK and deployment target. +An SDK file change consequently invalidates all Apple C/C++ profiles. Check the +input closure and actual native compiler with the pinned Xcode selected: + +```sh +export DEVELOPER_DIR="$(plutil -extract apple.developer_dir raw -o - tools/dev/native-host-toolchains.json)" +profile=//build_defs/host:native_macos_arm64 +bash tools/dev/bazel.sh test -c opt --config=macos-sandbox \ + //build_defs/host:apple_sdk_inputs_test //build_defs/host:host_c_smoke \ + --platforms="$profile" --host_platform="$profile" --jobs=2 +``` + +Extension inventories use the same stock Apple slice transition as their +XCFrameworks, so both consume the same configured object files. The desktop +member transition preserves its native deployment contract. On macOS, +`//build_defs/extensions:mobile_cpp_reuse_test` checks actual object-file reuse +across the macOS, iOS device and simulator inventories and frameworks. +It requires each branch to expose all three profiles before comparing their +object files, so losing framework traversal cannot produce a passing test. + +Full native qualifications and configured CI requests retain compact Bazel +execution logs in `diagnostics/execution-log.binpb.zst`. The workflow uploads +these beside the existing profiles, including on failure. Use the +[upstream execution-log parser/converter](https://github.com/bazelbuild/bazel/tree/8.8.1/src/tools/execlog) +from the pinned Bazel version to compare runs. Match actions by their outputs and +compare command, environment and complete input digests before attributing cache +misses to retention. An `aquery` ActionKey does not include input contents. +To check disk-cache reuse independently of Bazel's in-memory action cache, repeat +the identical request in a fresh output base with the same disk cache and retain +both execution logs. GitHub cache restore keys, source commits, runner context and +cache size before/after collection are also needed for a CI comparison. + +Cache transport and action reuse are separate measurements. A cancelled save +can leave the next job without a snapshot. GitHub scopes PR snapshots to that +PR, and repository-wide eviction competes with the legacy workflow caches. +The stock disk collector can also discard results before upload; inspect its +deleted-file/byte totals and the allocated-size bound in the save log. A larger +per-cohort disk budget does not increase GitHub's repository-wide capacity. + +Use `bazel config ` on configurations from the same +configured `aquery` to diagnose duplicate tools. In the Android graph, host +configurations can differ only by `@rules_android//rules:min_sdk_version`. +Pinned Bazel 8.8.1 propagates Starlark settings through the exec transition. +Its global exclusion option also resets Rust and Apple tool settings; avoid +enabling it without qualifying those consumers. Upstream's +[per-setting exec scope migration](https://github.com/bazelbuild/bazel/issues/26909) +provides the direction for a future Bazel upgrade. + Installed SDK checks also consume the completed dependencies declared by their manifests. With an existing GitHub CLI login, acquire their source-bound public input snapshot once: diff --git a/tools/ci/run-configured-request.mts b/tools/ci/run-configured-request.mts index 4c7b8b74a..20c83e7fd 100644 --- a/tools/ci/run-configured-request.mts +++ b/tools/ci/run-configured-request.mts @@ -183,12 +183,21 @@ export async function runConfiguredRequest(request: ConfiguredRequest, workspace ? { ...nativeBuildMetadata(source, request.github), oliphaunt_ci_request_id: request.id } : undefined; const bepFile = privateDirectory && path.join(privateDirectory, 'build-events.jsonl'); + const diagnostics = privateDirectory + ? requireSafeDirectoryChain(path.resolve(workspace, request.outputDirectory, 'diagnostics'), { + create: true, + label: 'CI build diagnostics', + }) + : undefined; const args = [ ...startup, request.command, ...request.targets, ...request.configuration, ...(bepFile ? [`--build_event_json_file=${bepFile}`] : []), + ...(diagnostics + ? [`--execution_log_compact_file=${path.join(diagnostics, 'execution-log.binpb.zst')}`] + : []), ...Object.entries(metadata ?? {}).map(([key, value]) => `--build_metadata=${key}=${value}`), ]; const result = spawnSync('bash', ['tools/dev/bazel.sh', ...args], { diff --git a/tools/ci/run-configured-request.test.mts b/tools/ci/run-configured-request.test.mts index d617c3d3f..1a615b0ad 100644 --- a/tools/ci/run-configured-request.test.mts +++ b/tools/ci/run-configured-request.test.mts @@ -265,6 +265,9 @@ test('plain full and ordinary requests use the real shared capture, source guard expect(receipt.mode).toBe(mode); expect(f.calls().filter((args) => args.includes('test'))).toHaveLength(1); if (mode === 'full') { + expect(f.calls()[0]).toContain( + `--execution_log_compact_file=${path.join(f.output, 'diagnostics/execution-log.binpb.zst')}`, + ); expect(f.calls()).toHaveLength(3); expect(receipt.projection.files).toEqual(['fixture/input.rs']); expect(receipt.projection.analysisInputs).toEqual(['fixture/BUILD']); diff --git a/tools/ci/workflow-moon-transfers.test.mts b/tools/ci/workflow-moon-transfers.test.mts index f986ff3ad..063cb45f1 100644 --- a/tools/ci/workflow-moon-transfers.test.mts +++ b/tools/ci/workflow-moon-transfers.test.mts @@ -75,14 +75,32 @@ if (!process.env.OLIPHAUNT_TRANSFER_FIXTURE_PHASE) '-m', 'fixture', ]); - // Moon tasks inherit their parent's workspace; this proof owns a separate one. - const env = { ...process.env, MOON_CACHE: 'off', MOON_WORKSPACE_ROOT: scratch }; + // This fixture owns its workspace and revisions, even inside a parent PR's CI. + const env = { + ...process.env, + MOON_CACHE: 'off', + MOON_WORKSPACE_ROOT: scratch, + CI: 'true', + GITHUB_ACTIONS: 'true', + GITHUB_REF: 'refs/pull/1/merge', + GITHUB_BASE_REF: 'parent-workspace-base', + }; delete env.MOON_BASE; delete env.MOON_HEAD; const run = () => spawnSync( process.env.MOON_BIN ?? 'moon', - ['run', 'fixture:qualify', '--upstream', 'none', '--no-actions'], + [ + 'run', + 'fixture:qualify', + '--upstream', + 'none', + '--no-actions', + '--base', + 'HEAD', + '--head', + 'HEAD', + ], { cwd: scratch, env, encoding: 'utf8' }, ); const result = run(); diff --git a/tools/release/bazel-shipping-inputs.mts b/tools/release/bazel-shipping-inputs.mts index 078a6fbea..6c349b289 100644 --- a/tools/release/bazel-shipping-inputs.mts +++ b/tools/release/bazel-shipping-inputs.mts @@ -370,7 +370,14 @@ export function buildShippingInputView( const visitedDepSetsByProduct = new Map>(); const actionDescriptions = new Map(); const sharedDefinitions = new Map(); - const physicalInputs = new Set(); + // Rank paths once so shared SDK closures sort as integers. Hashing still uses + // the same sorted, unique path strings, independently of artifact IDs/order. + const orderedPaths = [...new Set(artifactPaths.values())].sort(); + const pathRanks = new Map(orderedPaths.map((path, rank) => [path, rank])); + const artifactRanks = new Map( + [...artifactPaths].map(([artifact, path]) => [artifact, required(pathRanks, path)]), + ); + const physicalInputs = new Uint8Array(orderedPaths.length); const captureInputs = new Set(); const virtualOutputs = new Set(); const describeAction = (action: Row) => ({ @@ -397,8 +404,16 @@ export function buildShippingInputView( const actionDescription = (action: Row, inputs: Set) => { const cached = actionDescriptions.get(action); if (cached !== undefined) return cached; - const paths = [...new Set([...inputs].map((input) => required(artifactPaths, input)))].sort(); - for (const path of paths) physicalInputs.add(path); + const ranks = Uint32Array.from(inputs, (input) => required(artifactRanks, input)).sort(); + const paths: string[] = []; + let previous = -1; + for (const rank of ranks) { + if (rank !== previous) { + paths.push(orderedPaths[rank]); + physicalInputs[rank] = 1; + } + previous = rank; + } if (action.mnemonic === 'Middleman') for (const output of array(action.outputIds ?? [])) virtualOutputs.add(required(artifactPaths, id(output))); @@ -606,7 +621,9 @@ export function buildShippingInputView( return { schema: 'oliphaunt-bazel-construction-inputs-v2', products, - physicalInputs: [...physicalInputs].filter((path) => !virtualOutputs.has(path)).sort(), + physicalInputs: orderedPaths.filter( + (path, rank) => physicalInputs[rank] && !virtualOutputs.has(path), + ), captureInputs: [...captureInputs].sort(), }; } diff --git a/tools/release/bazel-shipping-inputs.test.mts b/tools/release/bazel-shipping-inputs.test.mts index 75e19c274..073680fe9 100644 --- a/tools/release/bazel-shipping-inputs.test.mts +++ b/tools/release/bazel-shipping-inputs.test.mts @@ -529,6 +529,40 @@ test('partial, substituted or cyclic graphs fail closed', () => { expect(() => buildShippingInputView(exported, graph)).toThrow('missing generated input producer'); }); +test('input identities preserve path ordering and aliases independently of graph IDs and unrelated paths', () => { + const { exported, graph } = fixture(); + graph.pathFragments.push( + { id: 8, parentId: 1, label: 'é.h' }, + { id: 9, parentId: 1, label: 'Alpha.h' }, + { id: 10, parentId: 1, label: '𐐀.h' }, + { id: 11, parentId: 1, label: 'unused' }, + ); + graph.artifacts.push( + { id: 5, pathFragmentId: 8 }, + { id: 4294967313, pathFragmentId: 9 }, + { id: 7, pathFragmentId: 10 }, + { id: 8, pathFragmentId: 2 }, + { id: 9, pathFragmentId: 11 }, + ); + graph.depSetOfFiles[0].directArtifactIds = [7, 8, 1, 4294967313, 5]; + const inputs = ['src/Alpha.h', 'src/alpha.c', 'src/é.h', 'src/𐐀.h']; + const view = buildShippingInputView(exported, graph); + const compile = view.products.alpha.actions + .map((action) => JSON.parse(action)) + .find((action) => action.mnemonic === 'CppCompile'); + expect(compile.inputs).toEqual({ + count: inputs.length, + sha256: createHash('sha256').update(JSON.stringify(inputs)).digest('hex'), + }); + expect(view.physicalInputs).toEqual(['bazel-out/alpha.o', ...inputs, 'tools/packager']); + expect(view.captureInputs).toEqual([...inputs, 'tools/packager']); + graph.pathFragments.at(-1)!.label = '!unused-first'; + graph.pathFragments.reverse(); + graph.artifacts.reverse(); + graph.depSetOfFiles[0].directArtifactIds.reverse(); + expect(buildShippingInputView(exported, graph)).toEqual(view); +}); + test('compact descriptions bind exact sorted physical input sets and retain the full capture closure', () => { const { exported, graph } = fixture(); const view = buildShippingInputView(exported, graph); diff --git a/tools/release/qualify-native-shipping.mts b/tools/release/qualify-native-shipping.mts index 89b9bcfd6..0d451e3cd 100644 --- a/tools/release/qualify-native-shipping.mts +++ b/tools/release/qualify-native-shipping.mts @@ -227,7 +227,7 @@ export async function qualifyNativeShipping( configuration.some( (item) => item === '--' || - /^--(?:build_metadata|build_event_json_file|output|output_file|output_groups|profile|query_file|starlark:file|starlark:expr|experimental_aquery_dump_after_build_format|experimental_aquery_dump_after_build_output_file)(?:=|$)/u.test( + /^--(?:build_metadata|build_event_json_file|execution_log_compact_file|execution_log_json_file|execution_log_binary_file|output|output_file|output_groups|profile|query_file|starlark:file|starlark:expr|experimental_aquery_dump_after_build_format|experimental_aquery_dump_after_build_output_file)(?:=|$)/u.test( item, ), ) @@ -493,6 +493,7 @@ export async function qualifyNativeShipping( ...metadata, `--build_event_json_file=${file('build-events.jsonl')}`, `--profile=${diagnosticFile('build-profile.json.gz')}`, + `--execution_log_compact_file=${diagnosticFile('execution-log.binpb.zst')}`, '--output_groups=+shipping_manifest,+shipping_policy,+observation_inputs', ]); assertSource(); diff --git a/tools/release/qualify-native-shipping.test.mts b/tools/release/qualify-native-shipping.test.mts index 15524d7ae..85e5d894c 100644 --- a/tools/release/qualify-native-shipping.test.mts +++ b/tools/release/qualify-native-shipping.test.mts @@ -806,6 +806,9 @@ for (const selection of ['explicit', 'export']) `--profile=${path.join(f.outputDirectory, 'diagnostics/build-profile.json.gz')}`, ); expect(builds[0]).toContain('//fixture:shipping'); + expect(builds[0]).toContain( + `--execution_log_compact_file=${path.join(f.outputDirectory, 'diagnostics/execution-log.binpb.zst')}`, + ); expect(builds[0]).toContain('//fixture:consumer'); expect(builds[0]).toContain( '--output_groups=+shipping_manifest,+shipping_policy,+observation_inputs', @@ -1355,6 +1358,10 @@ test('caller query substitution is rejected before any process even beyond OS ar f.request.configuration.push('--output_file=/tmp/substituted-query.json'); await expect(qualifyNativeShipping(f.request, f.workspace)).rejects.toThrow('cannot override'); expect(f.calls()).toEqual([]); + f.request.configuration.pop(); + f.request.configuration.push('--execution_log_compact_file=/tmp/substituted-log'); + await expect(qualifyNativeShipping(f.request, f.workspace)).rejects.toThrow('cannot override'); + expect(f.calls()).toEqual([]); } finally { f.cleanup(); }