diff --git a/.github/workflows/ost-source-ci.yml b/.github/workflows/ost-source-ci.yml index e81134d..3952df2 100644 --- a/.github/workflows/ost-source-ci.yml +++ b/.github/workflows/ost-source-ci.yml @@ -445,6 +445,265 @@ jobs: path: | ${{ matrix.bundle }}/.strata/reports/ .ost-ci/ + pr-workspace: + if: github.event_name == 'pull_request' + name: ${{ matrix.name }} + runs-on: ${{ matrix.runs_on }} + env: + OST_CI_CELL: ${{ matrix.name }} + OST_CI_LANE: ${{ matrix.lane }} + OST_CI_RUNNER_PROFILE: ${{ matrix.runner_profile }} + OST_CI_RUNS_ON: ${{ join(matrix.runs_on, ',') }} + OST_CI_RUNTIME_ARTIFACT: ${{ matrix.runtime_artifact }} + OST_CI_MINIMUM_TRUST: ${{ matrix.minimum_trust }} + OST_HOME: ${{ matrix.hosted && format('{0}/.ost-ci-home', github.workspace) || '' }} + strategy: + fail-fast: false + matrix: + include: + - name: workspace-pr-windows + lane: pull_request + runtime_artifact: sha256:c3ed40122756ea118166e1619efcaec463e7d2a42f6d978fe4e20b6b774c4b03 + target_trust: local + minimum_trust: local + require_evidence: all + evidence_flags: "--require-sbom --require-provenance" + platform: cy2026 + profile: usd + runs_on: ["windows-2022"] + hosted: true + runner_profile: windows-hosted + verify: test + runtime_remote: "oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:07cb84412017ece911adaed1175a6373865d863bc40124081cfc20c52de7f0d9" + host_python: "" + host_packages_apt: "" + host_packages_brew: "" + - name: workspace-pr-macos-arm64 + lane: pull_request + runtime_artifact: sha256:a9bb847ab5c7eb29d7425ff9acfb05b01c1751054d6a70628e48b06f8409a4a8 + target_trust: local + minimum_trust: local + require_evidence: all + evidence_flags: "--require-sbom --require-provenance" + platform: cy2026 + profile: usd + runs_on: ["macos-15"] + hosted: true + runner_profile: macos-arm64-hosted + verify: test + runtime_remote: "oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:dc0c980b868f91fe33e6f35a67a2bc5693b89f0243141c55ecb068ab225b0f3e" + host_python: "3.13" + host_packages_apt: "" + host_packages_brew: "" + - name: workspace-pr-linux + lane: pull_request + runtime_artifact: sha256:03f7d4ef263abb50511d17237e7cbdbe1b83dee02cff8b8f901f0484c7a7898e + target_trust: local + minimum_trust: local + require_evidence: all + evidence_flags: "--require-sbom --require-provenance" + platform: cy2026 + profile: usd + runs_on: ["ubuntu-24.04"] + hosted: true + runner_profile: linux-hosted + verify: test + runtime_remote: "oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:367a32bd1985ca9b07e8f7e64f95c1ded8dba16d68f17e0bffb09cec0b9dc3f6" + host_python: "3.13" + host_packages_apt: "libx11-dev libxt-dev libxext-dev libgl1-mesa-dev" + host_packages_brew: "" + steps: + - name: Check out the repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Hosted runner billing notice + if: ${{ matrix.hosted }} + shell: bash + run: echo "::notice title=OpenStrata hosted-runner usage::This job uses GitHub-hosted infrastructure. Private repositories may incur GitHub Actions usage charges. Review repository billing and Actions usage settings." + - name: Bootstrap ost 0.22.2 (pinned release asset, checksum-verified) + if: ${{ matrix.hosted }} + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + case "${RUNNER_OS}-${RUNNER_ARCH}" in + Linux-X64) triple=x86_64-unknown-linux-musl ; ext=tar.xz ;; + macOS-ARM64) triple=aarch64-apple-darwin ; ext=tar.xz ;; + macOS-X64) triple=x86_64-apple-darwin ; ext=tar.xz ;; + Windows-X64) triple=x86_64-pc-windows-msvc ; ext=zip ;; + *) echo "::error title=ost bootstrap::no ost release asset for ${RUNNER_OS}-${RUNNER_ARCH}" ; exit 1 ;; + esac + pinned="" + case "$triple" in + *) : ;; + esac + asset="ost-cli-${triple}.${ext}" + base="https://github.com/animu-sphere/open-strata/releases/download/v0.22.2" + curl -fsSLo "$asset" "$base/$asset" + curl -fsSLo "$asset.sha256" "$base/$asset.sha256" + actual="$( (command -v sha256sum > /dev/null && sha256sum "$asset" || shasum -a 256 "$asset") | cut -d' ' -f1 )" + published="$(cut -d' ' -f1 "$asset.sha256")" + if [ "$actual" != "$published" ]; then + echo "::error title=ost bootstrap::$asset hashes to $actual but the release publishes $published" ; exit 1 + fi + if [ -n "$pinned" ] && [ "$actual" != "$pinned" ]; then + echo "::error title=ost bootstrap::$asset hashes to $actual but the CI contract pins $pinned" ; exit 1 + fi + mkdir -p .ost-ci/bootstrap-bin + if [ "$ext" = "zip" ]; then + powershell -NoProfile -Command "Expand-Archive -LiteralPath '$asset' -DestinationPath '.ost-ci/bootstrap-bin' -Force" + else + tar -xf "$asset" -C .ost-ci/bootstrap-bin + fi + bin="$(find .ost-ci/bootstrap-bin -type f \( -name ost -o -name ost.exe \) | head -n 1)" + if [ -z "$bin" ]; then echo "::error title=ost bootstrap::no ost binary inside $asset" ; exit 1 ; fi + chmod +x "$bin" 2> /dev/null || true + bin_dir="$(cd "$(dirname "$bin")" && pwd)" + bin="$bin_dir/$(basename "$bin")" + executable="$bin" + exported_path="$bin_dir" + if [ "$RUNNER_OS" = "Windows" ]; then + if ! command -v cygpath > /dev/null; then + echo "::error title=ost bootstrap::cygpath is required to export a native Windows PATH" ; exit 1 + fi + executable="$(cygpath -w "$bin")" + exported_path="$(cygpath -w "$bin_dir")" + fi + echo "$exported_path" >> "$GITHUB_PATH" + json_executable="$(printf '%s' "$executable" | sed 's/\\/\\\\/g; s/"/\\"/g')" + json_exported_path="$(printf '%s' "$exported_path" | sed 's/\\/\\\\/g; s/"/\\"/g')" + printf '{"schema":1,"pinned_version":"%s","asset":"%s","sha256":"%s","executable":"%s","exported_path":"%s"}\n' "0.22.2" "$asset" "$actual" "$json_executable" "$json_exported_path" > .ost-ci/bootstrap.json + - name: Check ost is available and record its version + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + if [ "${{ matrix.hosted }}" = "true" ] && [ "$RUNNER_OS" = "Windows" ]; then + version="$(powershell -NoProfile -Command "& ost.exe --version" | tr -d '\r')" + else + version="$(ost --version)" + fi + + echo "$version" + if [ "${{ matrix.hosted }}" = "true" ] && [ "$version" != "ost 0.22.2" ]; then + echo "::error title=ost bootstrap::expected 'ost 0.22.2', got '$version'" ; exit 1 + fi + printf '{"schema":1,"ost_version":"%s"}\n' "$version" > .ost-ci/ost-version.json + - name: Validate the CI manifest + shell: bash + run: ost ci validate + - name: Restore the artifact registry cache (speed only, never correctness) + id: runtime-cache-restore + if: ${{ matrix.hosted && vars.OST_CI_DISABLE_CACHE != 'true' }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .ost-ci-home/artifacts + key: ost-registry-0.22.2-${{ runner.os }}-${{ runner.arch }}-${{ matrix.name }}-${{ matrix.runtime_artifact }} + - name: Pull the pinned runtime SDK from its remote reference + if: ${{ matrix.runtime_remote != '' }} + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + if ost artifact show "${{ matrix.runtime_artifact }}" --json > /dev/null 2>&1 \ + && ost artifact verify "${{ matrix.runtime_artifact }}" ${{ matrix.evidence_flags }} --json > .ost-ci/runtime-cache-verify.json; then + echo "pinned runtime already present and verified (cache hit) -- skipping the remote pull" + else + if [ "${{ matrix.hosted }}" = "true" ] && [ -n "${OST_HOME:-}" ]; then + rm -rf "${OST_HOME}/artifacts" + fi + ost artifact pull "${{ matrix.runtime_remote }}" --expect-artifact "${{ matrix.runtime_artifact }}" --require-kind runtime --json | tee .ost-ci/runtime-pull.json + fi + - name: Verify and materialize the pinned runtime SDK + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + printf '{"schema":1,"runtime_artifact":"%s","source":"%s"}\n' "${{ matrix.runtime_artifact }}" "${{ matrix.runtime_remote != '' && 'remote-pull' || 'local-registry' }}" > .ost-ci/runtime-source.json + ost artifact verify ${{ matrix.runtime_artifact }} --minimum-trust ${{ matrix.minimum_trust }} ${{ matrix.evidence_flags }} + ost runtime pull ${{ matrix.platform }} --profile ${{ matrix.profile }} --from-artifact ${{ matrix.runtime_artifact }} --force + - name: Remove resumable transfer state before caching + if: ${{ matrix.hosted && vars.OST_CI_DISABLE_CACHE != 'true' && steps.runtime-cache-restore.outputs.cache-hit != 'true' }} + shell: bash + run: rm -rf .ost-ci-home/artifacts/.partial-blobs + - name: Save the verified artifact registry cache + if: ${{ matrix.hosted && vars.OST_CI_DISABLE_CACHE != 'true' && steps.runtime-cache-restore.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .ost-ci-home/artifacts + key: ost-registry-0.22.2-${{ runner.os }}-${{ runner.arch }}-${{ matrix.name }}-${{ matrix.runtime_artifact }} + - name: Install the host packages this runtime needs to be consumed + if: ${{ matrix.host_packages_apt != '' || matrix.host_packages_brew != '' }} + shell: bash + env: + HOST_PACKAGES_APT: ${{ matrix.host_packages_apt }} + HOST_PACKAGES_BREW: ${{ matrix.host_packages_brew }} + run: | + set -euo pipefail + case "$RUNNER_OS" in + Linux) + packages="$HOST_PACKAGES_APT" + if [ -z "$packages" ]; then + echo "error: this cell declares host_packages but nothing under 'apt'; a Linux runner installs from the 'apt' list" >&2 + exit 1 + fi + sudo apt-get update + sudo apt-get install -y --no-install-recommends $packages + ;; + macOS) + packages="$HOST_PACKAGES_BREW" + if [ -z "$packages" ]; then + echo "error: this cell declares host_packages but nothing under 'brew'; a macOS runner installs from the 'brew' list" >&2 + exit 1 + fi + brew install $packages + ;; + *) + echo "error: host_packages has no installer for $RUNNER_OS; provision the dependency on the runner image instead" >&2 + exit 1 + ;; + esac + - name: Validate the materialized runtime (runnable tools) + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + ost runtime validate ${{ matrix.platform }} --profile ${{ matrix.profile }} --json | tee .ost-ci/runtime-validate.json + - name: Set up host Python for schema tooling + if: ${{ matrix.hosted && matrix.host_python != '' }} + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: ${{ matrix.host_python }} + - name: Record the schema-tooling Python contract + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + if [ "${{ matrix.hosted }}" = "true" ] && [ -n "${{ matrix.host_python }}" ]; then + source=host-setup-python + elif [ -n "${{ matrix.host_python }}" ]; then + source=operator-provisioned + else + source=runtime-bundled + fi + printf '{"schema":1,"host_python":"%s","source":"%s"}\n' "${{ matrix.host_python }}" "$source" > .ost-ci/python-setup.json + - name: Validate the workspace dependency graph + shell: bash + run: ost plugin test --workspace --graph-only --json + - name: Build the workspace from source + shell: bash + run: ost build --target ${{ matrix.platform }} --profile ${{ matrix.profile }} + - name: Run the workspace test suite + if: ${{ matrix.verify == 'test' }} + shell: bash + run: ost test --target ${{ matrix.platform }} --profile ${{ matrix.profile }} + - name: Upload the build logs and CI evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: report-${{ matrix.name }} + path: | + .strata/targets/ + .ost-ci/ mainline: if: github.event_name == 'push' name: ${{ matrix.name }} @@ -868,3 +1127,262 @@ jobs: path: | ${{ matrix.bundle }}/.strata/reports/ .ost-ci/ + mainline-workspace: + if: github.event_name == 'push' + name: ${{ matrix.name }} + runs-on: ${{ matrix.runs_on }} + env: + OST_CI_CELL: ${{ matrix.name }} + OST_CI_LANE: ${{ matrix.lane }} + OST_CI_RUNNER_PROFILE: ${{ matrix.runner_profile }} + OST_CI_RUNS_ON: ${{ join(matrix.runs_on, ',') }} + OST_CI_RUNTIME_ARTIFACT: ${{ matrix.runtime_artifact }} + OST_CI_MINIMUM_TRUST: ${{ matrix.minimum_trust }} + OST_HOME: ${{ matrix.hosted && format('{0}/.ost-ci-home', github.workspace) || '' }} + strategy: + fail-fast: false + matrix: + include: + - name: workspace-main-windows + lane: main + runtime_artifact: sha256:c3ed40122756ea118166e1619efcaec463e7d2a42f6d978fe4e20b6b774c4b03 + target_trust: local + minimum_trust: local + require_evidence: all + evidence_flags: "--require-sbom --require-provenance" + platform: cy2026 + profile: usd + runs_on: ["windows-2022"] + hosted: true + runner_profile: windows-hosted + verify: test + runtime_remote: "oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:07cb84412017ece911adaed1175a6373865d863bc40124081cfc20c52de7f0d9" + host_python: "" + host_packages_apt: "" + host_packages_brew: "" + - name: workspace-main-macos-arm64 + lane: main + runtime_artifact: sha256:a9bb847ab5c7eb29d7425ff9acfb05b01c1751054d6a70628e48b06f8409a4a8 + target_trust: local + minimum_trust: local + require_evidence: all + evidence_flags: "--require-sbom --require-provenance" + platform: cy2026 + profile: usd + runs_on: ["macos-15"] + hosted: true + runner_profile: macos-arm64-hosted + verify: test + runtime_remote: "oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:dc0c980b868f91fe33e6f35a67a2bc5693b89f0243141c55ecb068ab225b0f3e" + host_python: "3.13" + host_packages_apt: "" + host_packages_brew: "" + - name: workspace-main-linux + lane: main + runtime_artifact: sha256:03f7d4ef263abb50511d17237e7cbdbe1b83dee02cff8b8f901f0484c7a7898e + target_trust: local + minimum_trust: local + require_evidence: all + evidence_flags: "--require-sbom --require-provenance" + platform: cy2026 + profile: usd + runs_on: ["ubuntu-24.04"] + hosted: true + runner_profile: linux-hosted + verify: test + runtime_remote: "oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:367a32bd1985ca9b07e8f7e64f95c1ded8dba16d68f17e0bffb09cec0b9dc3f6" + host_python: "3.13" + host_packages_apt: "libx11-dev libxt-dev libxext-dev libgl1-mesa-dev" + host_packages_brew: "" + steps: + - name: Check out the repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Hosted runner billing notice + if: ${{ matrix.hosted }} + shell: bash + run: echo "::notice title=OpenStrata hosted-runner usage::This job uses GitHub-hosted infrastructure. Private repositories may incur GitHub Actions usage charges. Review repository billing and Actions usage settings." + - name: Bootstrap ost 0.22.2 (pinned release asset, checksum-verified) + if: ${{ matrix.hosted }} + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + case "${RUNNER_OS}-${RUNNER_ARCH}" in + Linux-X64) triple=x86_64-unknown-linux-musl ; ext=tar.xz ;; + macOS-ARM64) triple=aarch64-apple-darwin ; ext=tar.xz ;; + macOS-X64) triple=x86_64-apple-darwin ; ext=tar.xz ;; + Windows-X64) triple=x86_64-pc-windows-msvc ; ext=zip ;; + *) echo "::error title=ost bootstrap::no ost release asset for ${RUNNER_OS}-${RUNNER_ARCH}" ; exit 1 ;; + esac + pinned="" + case "$triple" in + *) : ;; + esac + asset="ost-cli-${triple}.${ext}" + base="https://github.com/animu-sphere/open-strata/releases/download/v0.22.2" + curl -fsSLo "$asset" "$base/$asset" + curl -fsSLo "$asset.sha256" "$base/$asset.sha256" + actual="$( (command -v sha256sum > /dev/null && sha256sum "$asset" || shasum -a 256 "$asset") | cut -d' ' -f1 )" + published="$(cut -d' ' -f1 "$asset.sha256")" + if [ "$actual" != "$published" ]; then + echo "::error title=ost bootstrap::$asset hashes to $actual but the release publishes $published" ; exit 1 + fi + if [ -n "$pinned" ] && [ "$actual" != "$pinned" ]; then + echo "::error title=ost bootstrap::$asset hashes to $actual but the CI contract pins $pinned" ; exit 1 + fi + mkdir -p .ost-ci/bootstrap-bin + if [ "$ext" = "zip" ]; then + powershell -NoProfile -Command "Expand-Archive -LiteralPath '$asset' -DestinationPath '.ost-ci/bootstrap-bin' -Force" + else + tar -xf "$asset" -C .ost-ci/bootstrap-bin + fi + bin="$(find .ost-ci/bootstrap-bin -type f \( -name ost -o -name ost.exe \) | head -n 1)" + if [ -z "$bin" ]; then echo "::error title=ost bootstrap::no ost binary inside $asset" ; exit 1 ; fi + chmod +x "$bin" 2> /dev/null || true + bin_dir="$(cd "$(dirname "$bin")" && pwd)" + bin="$bin_dir/$(basename "$bin")" + executable="$bin" + exported_path="$bin_dir" + if [ "$RUNNER_OS" = "Windows" ]; then + if ! command -v cygpath > /dev/null; then + echo "::error title=ost bootstrap::cygpath is required to export a native Windows PATH" ; exit 1 + fi + executable="$(cygpath -w "$bin")" + exported_path="$(cygpath -w "$bin_dir")" + fi + echo "$exported_path" >> "$GITHUB_PATH" + json_executable="$(printf '%s' "$executable" | sed 's/\\/\\\\/g; s/"/\\"/g')" + json_exported_path="$(printf '%s' "$exported_path" | sed 's/\\/\\\\/g; s/"/\\"/g')" + printf '{"schema":1,"pinned_version":"%s","asset":"%s","sha256":"%s","executable":"%s","exported_path":"%s"}\n' "0.22.2" "$asset" "$actual" "$json_executable" "$json_exported_path" > .ost-ci/bootstrap.json + - name: Check ost is available and record its version + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + if [ "${{ matrix.hosted }}" = "true" ] && [ "$RUNNER_OS" = "Windows" ]; then + version="$(powershell -NoProfile -Command "& ost.exe --version" | tr -d '\r')" + else + version="$(ost --version)" + fi + + echo "$version" + if [ "${{ matrix.hosted }}" = "true" ] && [ "$version" != "ost 0.22.2" ]; then + echo "::error title=ost bootstrap::expected 'ost 0.22.2', got '$version'" ; exit 1 + fi + printf '{"schema":1,"ost_version":"%s"}\n' "$version" > .ost-ci/ost-version.json + - name: Validate the CI manifest + shell: bash + run: ost ci validate + - name: Restore the artifact registry cache (speed only, never correctness) + id: runtime-cache-restore + if: ${{ matrix.hosted && vars.OST_CI_DISABLE_CACHE != 'true' }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .ost-ci-home/artifacts + key: ost-registry-0.22.2-${{ runner.os }}-${{ runner.arch }}-${{ matrix.name }}-${{ matrix.runtime_artifact }} + - name: Pull the pinned runtime SDK from its remote reference + if: ${{ matrix.runtime_remote != '' }} + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + if ost artifact show "${{ matrix.runtime_artifact }}" --json > /dev/null 2>&1 \ + && ost artifact verify "${{ matrix.runtime_artifact }}" ${{ matrix.evidence_flags }} --json > .ost-ci/runtime-cache-verify.json; then + echo "pinned runtime already present and verified (cache hit) -- skipping the remote pull" + else + if [ "${{ matrix.hosted }}" = "true" ] && [ -n "${OST_HOME:-}" ]; then + rm -rf "${OST_HOME}/artifacts" + fi + ost artifact pull "${{ matrix.runtime_remote }}" --expect-artifact "${{ matrix.runtime_artifact }}" --require-kind runtime --json | tee .ost-ci/runtime-pull.json + fi + - name: Verify and materialize the pinned runtime SDK + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + printf '{"schema":1,"runtime_artifact":"%s","source":"%s"}\n' "${{ matrix.runtime_artifact }}" "${{ matrix.runtime_remote != '' && 'remote-pull' || 'local-registry' }}" > .ost-ci/runtime-source.json + ost artifact verify ${{ matrix.runtime_artifact }} --minimum-trust ${{ matrix.minimum_trust }} ${{ matrix.evidence_flags }} + ost runtime pull ${{ matrix.platform }} --profile ${{ matrix.profile }} --from-artifact ${{ matrix.runtime_artifact }} --force + - name: Remove resumable transfer state before caching + if: ${{ matrix.hosted && vars.OST_CI_DISABLE_CACHE != 'true' && steps.runtime-cache-restore.outputs.cache-hit != 'true' }} + shell: bash + run: rm -rf .ost-ci-home/artifacts/.partial-blobs + - name: Save the verified artifact registry cache + if: ${{ matrix.hosted && vars.OST_CI_DISABLE_CACHE != 'true' && steps.runtime-cache-restore.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .ost-ci-home/artifacts + key: ost-registry-0.22.2-${{ runner.os }}-${{ runner.arch }}-${{ matrix.name }}-${{ matrix.runtime_artifact }} + - name: Install the host packages this runtime needs to be consumed + if: ${{ matrix.host_packages_apt != '' || matrix.host_packages_brew != '' }} + shell: bash + env: + HOST_PACKAGES_APT: ${{ matrix.host_packages_apt }} + HOST_PACKAGES_BREW: ${{ matrix.host_packages_brew }} + run: | + set -euo pipefail + case "$RUNNER_OS" in + Linux) + packages="$HOST_PACKAGES_APT" + if [ -z "$packages" ]; then + echo "error: this cell declares host_packages but nothing under 'apt'; a Linux runner installs from the 'apt' list" >&2 + exit 1 + fi + sudo apt-get update + sudo apt-get install -y --no-install-recommends $packages + ;; + macOS) + packages="$HOST_PACKAGES_BREW" + if [ -z "$packages" ]; then + echo "error: this cell declares host_packages but nothing under 'brew'; a macOS runner installs from the 'brew' list" >&2 + exit 1 + fi + brew install $packages + ;; + *) + echo "error: host_packages has no installer for $RUNNER_OS; provision the dependency on the runner image instead" >&2 + exit 1 + ;; + esac + - name: Validate the materialized runtime (runnable tools) + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + ost runtime validate ${{ matrix.platform }} --profile ${{ matrix.profile }} --json | tee .ost-ci/runtime-validate.json + - name: Set up host Python for schema tooling + if: ${{ matrix.hosted && matrix.host_python != '' }} + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: ${{ matrix.host_python }} + - name: Record the schema-tooling Python contract + shell: bash + run: | + set -euo pipefail + mkdir -p .ost-ci + if [ "${{ matrix.hosted }}" = "true" ] && [ -n "${{ matrix.host_python }}" ]; then + source=host-setup-python + elif [ -n "${{ matrix.host_python }}" ]; then + source=operator-provisioned + else + source=runtime-bundled + fi + printf '{"schema":1,"host_python":"%s","source":"%s"}\n' "${{ matrix.host_python }}" "$source" > .ost-ci/python-setup.json + - name: Validate the workspace dependency graph + shell: bash + run: ost plugin test --workspace --graph-only --json + - name: Build the workspace from source + shell: bash + run: ost build --target ${{ matrix.platform }} --profile ${{ matrix.profile }} + - name: Run the workspace test suite + if: ${{ matrix.verify == 'test' }} + shell: bash + run: ost test --target ${{ matrix.platform }} --profile ${{ matrix.profile }} + - name: Upload the build logs and CI evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: report-${{ matrix.name }} + path: | + .strata/targets/ + .ost-ci/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d34c1..8e0c63f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,57 +4,112 @@ All notable changes to this project are documented here. ## [Unreleased] +Nothing yet. + +## [0.10.0] - 2026-08-23 + +Resolver-backed source identity and external resolver interoperability. The +release record is [docs/releases/v0.10.0.md](docs/releases/v0.10.0.md). + +### Added + +- The resolver-neutral `ResolverAssetIdentity` contract and + `Stable` / `Unstable` / `Unavailable` classification API in `usdGeoCache`, + with the OpenUSD-facing adapter centralized in the shared authoring cache + bridge so no format reader extracts identity itself. +- `usdgeo::cache::CacheDecision`: seven stable, transport-neutral categories + that explain a cache decision, `CacheDecisionName` as the machine-matchable + form, fixed `CacheDecisionMessage` constants, and `IdentityDecision` to map an + identity stability onto its category. +- `HasSupersededIdentityEntry`, which distinguishes a changed validation + identity from a source never generated before without persisting an + identifier or a token. +- Cache-decision reporting through `TryLoadPointCloudCache`, projected onto four + COPC codes: `COPC009` reuse disabled, `COPC010` reuse permitted or taken, + `COPC011` identity changed, `COPC012` entry invalidated. Every message names + its exact category. +- `kind: workspace` CI cells on Windows, macOS, and Linux for both lanes, which + configure the repository root and run its CTest suite. They are what makes the + Tier 1 resolver contract gate a CI gate. +- `tools/tier2_fixture_server.py`, a loopback origin that honours `Range` and + logs every request, and `tools/tier2_resolver_integration.py`, the harness + that composes it with an external resolver and the COPC FileFormat. +- The recorded Tier 2 baseline in + [docs/reference/RESOLVER_BASELINE.md](docs/reference/RESOLVER_BASELINE.md), + against `usd-http-resolver` v0.4.0 and the 81 MB Autzen COPC. + +### Changed + +- Generated cache entries are addressed by a generation key and a source + identity key rather than one combined key. Revisions of one source are now + siblings under one generation directory, which is what makes + `resolver-identity-changed` reportable. The generation key carries caller + intent; source size, modification time, the resolved georeference, and the + new `Descriptor::sourceDerived` group - which the conversion tool's tile-plan + key moved into - carry what was read out of the source. +- `Invalidate` removes an emptied generation directory, so an invalidated cache + root does not accumulate empty parents. +- Removed standalone `httpresolver` product CI cells. The relocated test double + is built transitively by the COPC Tier 1 integration test, which the workspace + cells and the local gate both run. +- Added a shared cache-layout construction entry point so producer and consumer + tests derive resolver-backed cache entries from the same descriptor contract. + +### Fixed + +- Windows CTest registrations for the authoring bridge, converter, and + FileFormat integrations now prepend the OpenUSD imported `lib` and `bin` + directories and the configured Python runtime to `PATH`, so all linked + OpenUSD, TBB, and Python DLLs resolve in workspace CI without hiding host + runtime DLLs. +- The PLY tiled-read integration test now writes payloads beside its fixture, + keeping the generated payload references relative when a Windows workspace + and the system temporary directory are on different volumes. +- Resolver cache Tier 1 coverage verifies cache hits, incomplete and corrupted + entry invalidation, and validation-token changes through cache artifacts + instead of process-local counters that are not shared across a FileFormat DLL + boundary on Windows. + ### Documentation -- Recorded the v0.10.0 direction: resolver-backed source identity and external - resolver interoperability, with transport owned by the resolver. - Added the [resolver-backed source contract](docs/architecture/RESOLVER_SOURCE.md), covering the responsibility boundary, the transport-neutral `SourceIdentity` - model, `Stable` / `Unstable` / `Unavailable` identity classification, - generated-cache ownership and reuse rules, diagnostics categories, the - no-secrets rule, and the Tier 1 / Tier 2 test split. + model, identity classification, generated-cache ownership and reuse rules, + the diagnostics categories, the no-secrets rule, and the Tier 1 / Tier 2 test + split. Every section is now marked shipped or explicitly not implemented. - Stated that no resolver implementation is a build-time dependency, and that `usd-http-resolver` is one compatible implementation composed at runtime. - Relocated the repository-local resolver test double to - `tests/plugins/httpresolver` and documented that it is excluded from the - product surface and release matrix. -- Updated the Tier 2 plan for the released - [`usd-http-resolver`](https://github.com/animu-sphere/usd-http-resolver) - implementation and its resolver-neutral `ArAssetInfo` identity contract. + `tests/plugins/httpresolver` and documented its exclusion from the product + surface and release matrix. +- Recorded the cache layout change in + [MIGRATION.md](docs/compatibility/MIGRATION.md). - Added an OpenStrata 0.22.2 dogfooding record for the external resolver skeleton; it identifies repository setup work, not an OpenStrata defect. -### Added - -- Added the resolver-neutral `ResolverAssetIdentity` contract and - `Stable` / `Unstable` / `Unavailable` classification API to `usdGeoCache`. -- Added resolver identity conversion tests and cache-key invalidation coverage - for changed opaque validation tokens. - -The OpenUSD-facing resolver adapter is centralized in the shared authoring -cache bridge. Stable-identity generated-cache reuse and recovery are complete; -diagnostic completion and recorded external interoperability remain planned. - -### Changed +### Compatibility -- Removed standalone `httpresolver` product CI cells. The relocated test double - is built transitively by the COPC Tier 1 integration test in the root build, - keeping the local gate independent of external resolver repositories. Note - that Tier 1 is not yet part of the CI matrix: every declared cell builds a - single plugin bundle, where `USDGEO_BUILD_TESTS` is undefined, so neither the - fixture nor `pointcloudCopc_tests` is compiled there. Wiring Tier 1 into CI - is tracked as follow-up work. -- Added a shared cache-layout construction entry point so producer and - consumer tests derive resolver-backed cache entries from the same descriptor - contract. +- A `v0.9.0` cache root is never looked up under the new layout, so the first + run after upgrading is a miss that regenerates. Cache entries are derived + data; delete an old root to reclaim the space. +- `StableCacheKey`, `TryBuildLayout`, `Inspect`, `IsCacheHit`, and `Invalidate` + keep their signatures and meanings. Tooling that enumerated entries with a + single-level glob needs a second level. +- Existing LAS, LAZ, COPC, and PLY format ids, arguments, authored stage shape, + and fixed-grid tiling behavior remain compatible with v0.9.0. -### Fixed +### Known limitations -- Resolver cache Tier 1 coverage now verifies cache hits, incomplete and - corrupted entry invalidation, and validation-token changes through cache - artifacts instead of process-local counters that are not shared across a - FileFormat DLL boundary on Windows. +- Nothing publishes a generated cache entry for a COPC source: + `usd-pointcloud-convert` accepts `.las` and `.laz` local inputs only. Lookup, + the reuse rules, and the decision diagnostics are complete; a measurable + generated-cache hit ratio for a remote source waits on COPC generation. +- `usd-pointcloud-convert` does not accept resolver-addressable identifiers. +- The Tier 2 origin is loopback, so the recorded numbers are protocol and + selectivity numbers rather than latency numbers. +- Raw byte-range caching and its hit ratios belong to the resolver. +- COPC writing and new public USD schemas remain deferred. ## [0.9.0] - 2026-08-15 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d7778f..b9ab27b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,15 @@ endif() option(USDGEO_BUILD_BENCHMARKS "Build explicit OpenUSD streaming benchmark executables" OFF) +function(usdgeo_configure_openusd_test_runtime test_name) + if(WIN32) + string(REPLACE ";" "\\;" _usdgeo_host_path "$ENV{PATH}") + set_property(TEST "${test_name}" APPEND PROPERTY + ENVIRONMENT + "PATH=$\\;$/../bin\\;$\\;${_usdgeo_host_path}") + endif() +endfunction() + set(_usdgeo_build_usd_default OFF) if(DEFINED pxr_ROOT OR DEFINED OpenUSD_ROOT) set(_usdgeo_build_usd_default ON) diff --git a/README.md b/README.md index ccf49e2..bb498be 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,12 @@ and tests without it. The repository-local fixture in memory; it is not a network transport or product bundle. Generated-USDC cache reuse for resolver-backed sources requires stable source -identity and stays disabled when the resolver cannot supply it. The boundary, -the identity model, and the planned `v0.10.0` work are in the -[resolver-backed source contract](docs/architecture/RESOLVER_SOURCE.md). +identity and stays disabled when the resolver cannot supply it, with the reason +reported through a stable category rather than silently. The boundary and the +identity model are in the +[resolver-backed source contract](docs/architecture/RESOLVER_SOURCE.md); what a +remote read actually costs is recorded in the +[resolver read baseline](docs/reference/RESOLVER_BASELINE.md). ## Quick Start @@ -287,8 +290,10 @@ dataset coverage remains open. `ConflictingCrs` diagnostic. - The deterministic USDC cache is available to the conversion tool through `--cache-root`; direct FileFormat lookup reuses committed entries through - `USDGEO_CACHE_ROOT`. Resolver-backed sources are excluded from reuse until a - stable source identity is available; enabling that case is `v0.10.0` work. + `USDGEO_CACHE_ROOT`. Reuse requires either a stable local filesystem identity + or a `Stable` resolver identity, and fails closed otherwise. Only the + conversion tool publishes entries, and it accepts `.las` and `.laz` local + inputs, so a COPC read has nothing to reuse yet. - HTTP, cloud SDKs, authentication, retries, and raw byte-range caching are out of scope; they belong to the resolver implementation. - Writing LAS, LAZ, or COPC is out of scope; all three plugins export as @@ -299,16 +304,20 @@ See the [implementation status](docs/roadmap/implementation-status.md) and ## Status -Latest release: **v0.9.0** — TilePlan convergence across sequential and COPC -native planning, plus a reproducible host-responsiveness baseline. The v0.3.0 module and -bundle rename is recorded in [MIGRATION.md](docs/compatibility/MIGRATION.md). -See the [release record](docs/releases/v0.9.0.md) and [CHANGELOG.md](CHANGELOG.md). - -Next: **v0.10.0** — resolver-backed source identity and external resolver -interoperability, so generated output can be reused safely for -resolver-provided sources while transport stays with the resolver. Scope and -exit gate are in the -[infrastructure maturity roadmap](docs/roadmap/infrastructure-maturity.md). +Latest release: **v0.10.0** — generated-cache decisions explained through a +stable transport-neutral vocabulary, a revision-aware cache layout, the Tier 1 +resolver contract gate running in CI on every host, and recorded +interoperability with a released external resolver. The cache layout change is +recorded in [MIGRATION.md](docs/compatibility/MIGRATION.md), along with the +v0.3.0 module and bundle rename. See the +[release record](docs/releases/v0.10.0.md) and +[CHANGELOG.md](CHANGELOG.md). + +Next: format-independent depth continues over format count. The nearest open +items are publishing generated cache entries for COPC sources and accepting +resolver-addressable identifiers in the conversion tool, both of which the +[infrastructure maturity roadmap](docs/roadmap/infrastructure-maturity.md) +places before any new format. Direction is fixed in the [design policy](docs/design/DESIGN_POLICY.md); the structure is fixed in the diff --git a/VERSION b/VERSION index ac39a10..78bc1ab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9.0 +0.10.0 diff --git a/docs/README.md b/docs/README.md index 9167c40..b425a06 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,7 +12,7 @@ workspace contract wins; structural changes must update that contract first. | Category | Answers | Start here | | --- | --- | --- | | [architecture/](architecture/) | How the workspace is structured, which dependency directions are legal, and what each cross-cutting contract requires. | [WORKSPACE.md](architecture/WORKSPACE.md) | -| [reference/](reference/) | What point-cloud input is accepted today and how it maps to USD. | [CAPABILITY_MATRIX.md](reference/CAPABILITY_MATRIX.md), [POINTCLOUD_METADATA.md](reference/POINTCLOUD_METADATA.md) | +| [reference/](reference/) | What point-cloud input is accepted today, how it maps to USD, and what a resolver-backed read costs. | [CAPABILITY_MATRIX.md](reference/CAPABILITY_MATRIX.md), [POINTCLOUD_METADATA.md](reference/POINTCLOUD_METADATA.md), [RESOLVER_BASELINE.md](reference/RESOLVER_BASELINE.md) | | [guides/](guides/) | How to build, test, install, and redistribute the plugins. | [BUILDING.md](guides/BUILDING.md), [INSTALL.md](guides/INSTALL.md) | | [compatibility/](compatibility/) | Which OpenUSD and OpenStrata versions are supported, and how to migrate across renames. | [OPENUSD.md](compatibility/OPENUSD.md), [MIGRATION.md](compatibility/MIGRATION.md) | | [roadmap/](roadmap/) | What remains incomplete and in what order it lands. | [README.md](roadmap/README.md) | @@ -47,6 +47,9 @@ workspace contract wins; structural changes must update that contract first. boundary toward external resolvers: resolver-backed byte access, transport-neutral source identity, generated-cache ownership, and the diagnostics that explain a cache decision. +- [reference/RESOLVER_BASELINE.md](reference/RESOLVER_BASELINE.md) records the + Tier 2 numbers for reading a COPC asset through an external resolver: request + counts, bytes fetched over source size, and local/remote output equivalence. - [architecture/PLUGIN_ADAPTER.md](architecture/PLUGIN_ADAPTER.md) is the thin-adapter rule every FileFormat Plugin is held to. - [architecture/DIAGNOSTICS.md](architecture/DIAGNOSTICS.md) defines the typed diff --git a/docs/architecture/DIAGNOSTICS.md b/docs/architecture/DIAGNOSTICS.md index a77966e..c614a9b 100644 --- a/docs/architecture/DIAGNOSTICS.md +++ b/docs/architecture/DIAGNOSTICS.md @@ -21,9 +21,25 @@ Codes are owned per plugin and listed in [pointcloud-las diagnostics](../../plugins/pointcloud-las/docs/DIAGNOSTICS.md) and [pointcloud-laz diagnostics](../../plugins/pointcloud-laz/docs/DIAGNOSTICS.md). The existing import-stage codes are fatal, because none of them leave a stage -that can be opened. Resolver-backed cache decisions are recoverable warnings; -COPC reports disabled reuse with `COPC009` and continues by reading and -authoring from the source. +that can be opened. + +Generated-cache decisions are the exception: they are recoverable, because a +cache decision changes what is reused and never what is read. `usdGeoCache` +owns their vocabulary as `usdgeo::cache::CacheDecision`, whose seven stable +category names are listed in the +[resolver-backed source contract](RESOLVER_SOURCE.md). COPC projects them onto +four codes, and every emitted message names its exact category: + +| Code | Severity | Categories | +| --- | --- | --- | +| `COPC009` | warning | `resolver-identity-unavailable`, `resolver-identity-unstable`, `generated-cache-reuse-disabled` | +| `COPC010` | status | `resolver-identity-stable`, `generated-cache-hit` | +| `COPC011` | status | `resolver-identity-changed` | +| `COPC012` | warning | `generated-cache-invalidated` | + +Category names obey rule 1 below: a name is never reused for a different +meaning. Decision messages are fixed constants owned by `usdgeo::cache`, so no +transport specific and no token content can reach one. Remaining limitations of the current migration: diff --git a/docs/architecture/RESOLVER_SOURCE.md b/docs/architecture/RESOLVER_SOURCE.md index 2d35283..13501b0 100644 --- a/docs/architecture/RESOLVER_SOURCE.md +++ b/docs/architecture/RESOLVER_SOURCE.md @@ -9,9 +9,10 @@ Structure belongs to [WORKSPACE.md](WORKSPACE.md); this document owns the resolver-facing behavior those modules implement. Cache layout and invalidation belong to the [`usdGeoCache` README](../../libs/usd-geo-cache/README.md). -Sections marked **Planned (`v0.10.0`)** are direction, not shipped behavior. -What the tree implements today is in -[CAPABILITY_MATRIX.md](../reference/CAPABILITY_MATRIX.md). +Everything below is shipped as of `v0.10.0` unless a section says otherwise. +What the tree implements is in +[CAPABILITY_MATRIX.md](../reference/CAPABILITY_MATRIX.md); the recorded Tier 2 +numbers are in [RESOLVER_BASELINE.md](../reference/RESOLVER_BASELINE.md). ## 1. Responsibility boundary @@ -168,15 +169,20 @@ source revision B, same identifier, different validation token Resolver-backed FileFormat reads stay preview and inspection paths. They reuse generated cache only under stable identity, and they diagnose explicitly when -reuse is disabled rather than silently regenerating. +reuse is disabled rather than silently regenerating. This is shipped: the COPC +FileFormat consumes a committed entry under `Stable` identity and reports every +other outcome through the categories in §4. `usd-pointcloud-convert` remains the production path for deterministic, -long-running payload generation. For resolver-backed inputs it accepts -resolver-addressable identifiers where the active OpenUSD environment supports -them, computes the same resolver-neutral identity, populates and reuses the -generated cache only under stable identity, and may record the normalized -identity class in manifest or debug metadata — never the transport secrets -covered in §2.3. +long-running payload generation, and it is the only thing that publishes a +generated entry. **Not implemented (`v0.10.0`):** it accepts `.las` and `.laz` +local inputs only. No COPC input, and no resolver-addressable identifier, +reaches it, so no COPC read — local or resolver-backed — has an entry to hit in +a normal workflow. The lookup side is complete and covered; the generation side +for COPC and for resolver-addressable inputs is future work. When it lands it +computes the same resolver-neutral identity, populates and reuses the generated +cache only under stable identity, and may record the normalized identity class +in manifest or debug metadata — never the transport secrets covered in §2.3. ### 3.3 Cache ownership boundary — Implemented (`v0.10.0`) @@ -185,25 +191,59 @@ The generated-USDC cache is owned by `usdGeoCache` and manifest, and generated payloads, and their identity includes the resolver validation token. +An entry's path is two levels, and both components are 64-bit hashes: + +```text +/// +``` + +The split follows one rule: caller intent chooses the directory, and everything +read out of the source chooses the entry inside it. The generation key covers +the resolved identifier, the plugin, parser, and OpenUSD versions, attribute +selection, tiling and LOD arguments including planner identity and version, and +downsampling. The source identity key covers the revision metadata - size, +modification time, and the opaque validation token - plus the georeference +resolved from the source header and any plan computed by scanning it. + +The georeference belongs in the second half because it is source-derived: the +local origin is the source bounding box, and the CRS may be an embedded record. +Putting it in the first half would mean a revision whose bounding box moved +landed in an unrelated generation directory, where nothing could see that it +superseded anything. The caller's *explicit* coordinate arguments are a +different value and stay in the generation key with the rest of the normalized +arguments. + +Two consequences follow, and both are load-bearing. Revisions of one source +collect side by side under one generation directory, which is how a changed +validation token is reported as `resolver-identity-changed` rather than as a +source never seen before. And neither level renders an identifier or a token, so +a signed URL cannot be read back out of a cache root — see §2.3. + Source byte-range caching is a separate concern owned by the active resolver and its `ArAsset` implementation. The point-cloud readers and `ArAssetRandomAccessSource` perform bounded reads but do not persist source ranges in the generated-USDC cache. This prevents transport or resolver fetch state from becoming an implicit generated-asset cache key or artifact. -## 4. Diagnostics — Planned (`v0.10.0`) +## 4. Diagnostics — Implemented (`v0.10.0`) -Cache decisions are explained through stable categories: +Cache decisions are explained through stable categories. `usdgeo::cache` +publishes them as `CacheDecision`, and `CacheDecisionName` is the string form a +consumer matches on: -```text -resolver identity unavailable -resolver identity unstable -resolver identity stable -resolver identity changed -generated cache reuse disabled -generated cache hit -generated cache invalidated -``` +| Category | Name | +| --- | --- | +| resolver identity unavailable | `resolver-identity-unavailable` | +| resolver identity unstable | `resolver-identity-unstable` | +| resolver identity stable | `resolver-identity-stable` | +| resolver identity changed | `resolver-identity-changed` | +| generated cache reuse disabled | `generated-cache-reuse-disabled` | +| generated cache hit | `generated-cache-hit` | +| generated cache invalidated | `generated-cache-invalidated` | + +Names are stable once published, in the sense rule 1 of the +[diagnostics contract](DIAGNOSTICS.md) defines: a name is never reused for a +different meaning. Messages are for humans and may change between releases. Messages describe the decision without leaking transport specifics or token contents: @@ -214,8 +254,23 @@ a stable source validation identity. ``` `Missing HTTP ETag` is not an acceptable message, because HTTP is not part of -this contract. Codes project onto the existing prefixes described in the -[diagnostics contract](DIAGNOSTICS.md). +this contract. Every message is a fixed constant owned by `usdgeo::cache`, so a +transport detail cannot reach one by accident, and a unit test asserts that none +of them names one. + +Codes project onto the existing prefixes described in the +[diagnostics contract](DIAGNOSTICS.md). The COPC projection is four codes over +the seven categories, and every emitted message names its exact category: + +| Code | Severity | Categories | +| --- | --- | --- | +| `COPC009` | warning | identity unavailable, identity unstable, reuse disabled | +| `COPC010` | status | identity stable, cache hit | +| `COPC011` | status | identity changed | +| `COPC012` | warning | cache invalidated | + +`resolver-identity-changed` is observable because the generated-cache layout +separates what would be generated from which revision was read; see §3.3. ## 5. Interoperability @@ -241,16 +296,20 @@ pointcloud-copc `usd-http-resolver` is one compatible implementation, not a required dependency. Registration is in [INSTALL.md](../guides/INSTALL.md). -## 6. Testing tiers — In progress (`v0.10.0`) +## 6. Testing tiers — Implemented (`v0.10.0`) **Tier 1 — repository-local contract tests.** They run with no external -resolver repository, using fake or memory-backed test assets, and remain the -required local gate; CI wiring is outstanding. Coverage: resolver-backed random access, partial reads, -short-read diagnostics, stable / unstable / unavailable identity, miss-to-hit -behavior, invalidation on validation-token change, corruption recovery, -`TilePlan` compatibility in cache keys, and deterministic diagnostics. - -**Tier 2 — cross-repository integration.** An external resolver, a local +resolver repository, using fake or memory-backed test assets, and are the +required gate on every host and both lanes. `openstrata.ci.yaml` declares +`kind: workspace` cells that configure the repository root — where +`USDGEO_BUILD_TESTS` defaults to `ON` — and run its CTest suite; a per-plugin +bundle cell cannot compile these tests. Coverage: resolver-backed random access, +partial reads, short-read diagnostics, stable / unstable / unavailable identity, +miss-to-hit behavior, invalidation on validation-token change, superseded-entry +detection, corruption recovery, `TilePlan` compatibility in cache keys, and +deterministic diagnostics. + +**Tier 2 — cross-repository integration — recorded (`v0.10.0`).** An external resolver, a local reproducible HTTP server, and a COPC fixture verify that a URL resolves, that metadata, hierarchy, and point-range reads succeed, that authored local and resolver-backed output is equivalent, that stable identity enables reuse, and @@ -260,11 +319,18 @@ OpenStrata workspace composition without making this repository structurally dependent on the resolver repository. [`usd-http-resolver`](https://github.com/animu-sphere/usd-http-resolver) is the -designated first Tier 2 implementation. Its `v0.2.0` release provides the HTTP -backend and OpenUSD resolver bundle, exposes stable resolver-neutral identity -through `ArAssetInfo`, and is tested through its own OpenStrata workflow. Tier -1 remains this repository's required local gate; Tier 2 is now ready to be -composed and recorded as the `v0.10.0` release gate. +first Tier 2 implementation, and `v0.10.0` is recorded against its `v0.4.0` +release, which provides the HTTP backend, the OpenUSD resolver bundle, and +resolver-neutral identity through `ArAssetInfo`. +`tools/tier2_fixture_server.py` is the loopback origin and +`tools/tier2_resolver_integration.py` the harness; the numbers are in +[RESOLVER_BASELINE.md](../reference/RESOLVER_BASELINE.md). + +The recorded run shows a metadata open costing 0.15% of an 81 MB asset, a full +read costing exactly 1.0, local and resolver-backed reads authoring the same +10,653,336 points under the same digest, a strong validator classifying as +`Stable`, and a weak validator classifying as `Unstable` and disabling reuse +while authoring identical output. ## 7. Test-double resolver diff --git a/docs/architecture/WORKSPACE.md b/docs/architecture/WORKSPACE.md index a46caa4..5dff0ee 100644 --- a/docs/architecture/WORKSPACE.md +++ b/docs/architecture/WORKSPACE.md @@ -30,7 +30,7 @@ modules implement is fixed in | `pointcloud-copc` | `plugins/pointcloud-copc` | OpenStrata plugin bundle (`usd-fileformat`) | implemented (resolver-backed read) | Resolver-opened `ArAsset` adaptation, metadata-only and non-tiled reads, and native hierarchy tiled COPC authoring through shared `usdLod`. Remote tiled reads require a local payload directory; source point ranges remain unsupported. | | `httpresolver` | `tests/plugins/httpresolver` | test-only OpenUSD `ArResolver` fixture | Tier 1 fixture | OpenUSD `ArResolver` test double for `http://memory.copc` and `https://memory.copc`, serving a configured local fixture as an in-memory `ArAsset`. It has no network transport or product bundle manifest, is built only with COPC integration tests, and is excluded from product discovery and release matrices. | | `usdPointCloudTiling` | `libs/usd-pointcloud-tiling` | plain CMake/OpenStrata static library | implemented | Format-independent fixed-grid partitioning, spill-backed bounded-memory routing, deterministic tile and LOD ordering, validated tile manifest serialization, spool validation, and cleanup contracts. See the [streaming and tiling plan](../roadmap/streaming-and-tiling.md). | -| `usdGeoCache` | `libs/usd-geo-cache` | plain CMake/OpenStrata static library | implemented | Descriptor-based stable cache keys, deterministic USDC root/payload layout, machine-readable lookup states, process-local lookup statistics, and entry invalidation. The conversion tool owns generation and atomic publication; direct FileFormat adapters reuse committed entries through `USDGEO_CACHE_ROOT`. | +| `usdGeoCache` | `libs/usd-geo-cache` | plain CMake/OpenStrata static library | implemented | Descriptor-based stable cache keys, deterministic USDC root/payload layout, machine-readable lookup states, process-local lookup statistics, the stable cache-decision vocabulary, and entry invalidation. Entries are addressed by a generation key over what the caller asked for and a source identity key over everything read out of the source, so revisions of one source are siblings. The conversion tool owns generation and atomic publication; direct FileFormat adapters reuse committed entries through `USDGEO_CACHE_ROOT`. | | `usdPly` | `libs/usd-ply` | plain CMake/OpenStrata static library | implemented | PLY 1.0 header inspection, scalar vertex decoding, source filters, and explicit georeference conversion into shared point-cloud assets. | | `usdAsciiPoints`, `usdE57` | `libs/` | plain libraries | reserved, not implemented | Additional point-cloud readers targeting the same shared contracts. | | `pointcloud-ply` | `plugins/pointcloud-ply` | OpenStrata plugin bundle (`usd-fileformat`) | implemented | Thin `.ply` adapter requiring an explicit `epsg` argument and authoring through shared point-cloud contracts. | @@ -267,16 +267,21 @@ Every structural or format change preserves these invariants: `openstrata.ci.yaml` is the source of truth; the GitHub workflow is generated by `ost ci generate github`. The declared PR matrix runs every production -bundle on every host. Every cell is a per-plugin bundle build, so the Tier 1 -resolver test double has no product-bundle cell and is not built by CI; it is -built by the root `ost build` in the local gate below, where -`USDGEO_BUILD_TESTS` defaults to `ON`: - -| Host | Target | OST level | -| --- | --- | --- | -| Windows 2022 x86_64 | cy2026 / USD | L0-L4 | -| macOS 15 arm64 | cy2026 / USD | L0-L5 | -| Ubuntu 24.04 x86_64 | cy2026 / USD | L0-L5 | +bundle on every host, plus a `kind: workspace` cell per host on both lanes. + +The two cell kinds exist for different reasons. A bundle cell builds one +plugin from `plugins/pointcloud-/CMakeLists.txt`, a standalone `project()` +that never declares `USDGEO_BUILD_TESTS`, so it cannot compile the repository's +test suite or the Tier 1 resolver test double. A workspace cell configures the +repository root, where `USDGEO_BUILD_TESTS` defaults to `ON`, and runs its +CTest suite. The workspace cells are therefore what makes the Tier 1 resolver +contract gate a CI gate rather than a local-only one: + +| Host | Target | Bundle cells | Workspace cell | +| --- | --- | --- | --- | +| Windows 2022 x86_64 | cy2026 / USD | L0-L4 | `ost build` + `ost test` | +| macOS 15 arm64 | cy2026 / USD | L0-L5 | `ost build` + `ost test` | +| Ubuntu 24.04 x86_64 | cy2026 / USD | L0-L5 | `ost build` + `ost test` | The required local gate is: @@ -292,20 +297,23 @@ ost plugin test plugins/pointcloud-laz --up-to 4 ost plugin test plugins/pointcloud-copc --up-to 4 ``` -The LAS, LAZ, COPC, and PLY bundles declare OST smoke fixtures and run the L3 -`usdcat.read` and L4 `python.stage_open` checks. The test-only `httpresolver` -bundle has no standalone fixture or CI matrix cell; its functional path is -exercised by the COPC Tier 1 integration test in the root build, which the -local gate runs and CI does not. The COPC bundle follows the -same runtime matrix as LAS and LAZ. +The LAS, LAZ, and PLY bundles declare OST smoke fixtures and run the L3 +`usdcat.read` and L4 `python.stage_open` checks. The `pointcloud-copc` bundle +declares none, so those two levels skip for it and its L2 discovery check is +the extent of its bundle-cell coverage; its functional coverage is the Tier 1 +integration test the workspace cells run. Declaring a checked-in COPC smoke +fixture is open work. The test-only `httpresolver` +bundle has no standalone fixture and no bundle cell; it is built transitively by +the COPC Tier 1 integration test, which the workspace cells and the local gate +both run. The COPC bundle follows the same runtime matrix as LAS and LAZ. The gate must stay passable without any external resolver repository. From `v0.10.0`, repository-local resolver contract tests (Tier 1) are the required -*local* gate; wiring them into the CI matrix is still outstanding, since the -declared cells build plugin bundles individually rather than the repository -root. Cross-repository integration against an external resolver implementation -(Tier 2) is composed separately; see -[RESOLVER_SOURCE.md](RESOLVER_SOURCE.md). +gate on every host and both lanes, carried by the workspace cells above. +Cross-repository integration against an external resolver implementation +(Tier 2) is composed at runtime and recorded separately; see +[RESOLVER_SOURCE.md](RESOLVER_SOURCE.md) §6 and +[RESOLVER_BASELINE.md](../reference/RESOLVER_BASELINE.md). ## 10. Delivery status @@ -321,6 +329,7 @@ root. Cross-repository integration against an external resolver implementation | v0.7.0 | point-budget-aware adaptive tiling | released 2026-08-13 | | v0.8.0 | real-world measurement and I/O observability | released 2026-08-14 | | v0.9.0 | TilePlan convergence and interactive validation | released 2026-08-15 | +| v0.10.0 | resolver-backed source identity, generated-cache decision diagnostics, Tier 1 as a CI gate, and recorded external resolver interoperability | released 2026-08-23 | Current work and acceptance gaps are tracked in [roadmap/implementation-status.md](../roadmap/implementation-status.md). diff --git a/docs/compatibility/MIGRATION.md b/docs/compatibility/MIGRATION.md index ec6d14e..ecb6d19 100644 --- a/docs/compatibility/MIGRATION.md +++ b/docs/compatibility/MIGRATION.md @@ -4,6 +4,49 @@ Breaking changes to names, paths, and identifiers, and what to do about them. The project is pre-1.0, so cleanup is appropriate — but every migration is recorded here explicitly rather than left for a consumer to discover. +## v0.10.0: generated cache entry layout + +A generated cache entry moved from one directory below the cache root to two: + +```text +# before +// +# after +/// +``` + +The generation key covers what the caller asked for; the source identity key +covers everything read out of the source - size, modification time, the +validation token, the georeference resolved from the header, and any plan +computed by scanning it. Both are the same 16-hex-character hash as before. + +Why: with one key, a source read at a new revision produced an unrelated +directory, so a changed validation token was indistinguishable from a source +never generated before. Under two, revisions of one source are siblings, which +is what the `resolver-identity-changed` diagnostic reports. + +What to do: nothing, in the normal case. Cache entries are derived data and a +`v0.9.0` root is simply never looked up, so the first run after upgrading is a +miss that regenerates. Delete an old root to reclaim the space: + +```powershell +Remove-Item -Recurse $env:USDGEO_CACHE_ROOT +``` + +Anything that enumerated entries with a single-level glob needs a second level. +The conversion conformance test is the in-tree example: + +```cmake +# before +file(GLOB cache_entries RELATIVE "${cache_root}" "${cache_root}/*") +# after +file(GLOB cache_entries RELATIVE "${cache_root}" "${cache_root}/*/*") +``` + +No authored output, file-format argument, USD attribute, or diagnostic code +changes. `StableCacheKey`, `TryBuildLayout`, `Inspect`, `IsCacheHit`, and +`Invalidate` keep their signatures and their meanings. + ## Unreleased: repository rename The GitHub repository was renamed from `animu-sphere/usd-geo-plugins` to diff --git a/docs/reference/CAPABILITY_MATRIX.md b/docs/reference/CAPABILITY_MATRIX.md index f7d870d..896af05 100644 --- a/docs/reference/CAPABILITY_MATRIX.md +++ b/docs/reference/CAPABILITY_MATRIX.md @@ -34,7 +34,9 @@ uses the same reader and authoring contracts for long-running generation. | COPC point-data decoding | Supported | Local hierarchy ranges are decoded as LAZ chunks through `usdlaz::DecodeLazChunk`; bounds, classification, and attribute selection use the shared point-cloud contracts | | COPC FileFormat Plugin | Supported | `pointcloud-copc` provides local metadata-only, non-tiled, and native hierarchy tiled reads; tiled output is payload-backed `usdLod`, while source point ranges are rejected because hierarchy order is spatial | | Resolver-backed COPC reads | Supported | The plugin adapts an `ArAsset` opened through the active `ArResolver` to the project-owned random-access source; remote tiled reads require an absolute local `payloadDirectory`. Transport, authentication, retries, and raw byte caching belong to the resolver | -| Resolver-backed generated-cache reuse | Foundation | COPC now extracts resolver-neutral identity through the shared adapter and attempts lookup only for `Stable` identity; complete stable-identity generation, corruption recovery, and cross-resolver coverage remain in v0.10.0 work; see the [resolver-backed source contract](../architecture/RESOLVER_SOURCE.md) | +| Resolver-backed generated-cache lookup | Supported | COPC extracts resolver-neutral identity through the shared adapter and reuses a committed entry only for `Stable` identity. Incomplete and corrupted entries are invalidated, and a changed validation token regenerates rather than hitting the superseded entry. Recorded against an external resolver in the [resolver read baseline](RESOLVER_BASELINE.md); the contract is the [resolver-backed source contract](../architecture/RESOLVER_SOURCE.md) | +| Resolver-backed generated-cache generation | Not supported | Entries are published by `usd-pointcloud-convert`, which accepts `.las` and `.laz` local inputs only. No COPC read - local or resolver-backed - has an entry to hit in a normal workflow | +| Generated-cache decision diagnostics | Supported | `usdgeo::cache::CacheDecision` publishes seven stable, transport-neutral categories; COPC projects them onto `COPC009`-`COPC012` and every message names its category | ## LAS Versions @@ -243,8 +245,11 @@ in the [tile and LOD contract](../architecture/LOD.md). passes before payload authoring. The conversion tool can generate and reuse deterministic USDC entries with `--cache-root`; direct LAS, LAZ, COPC, and PLY FileFormat lookup reuses committed entries - through `USDGEO_CACHE_ROOT`. Reuse requires a stable local filesystem - identity; resolver-backed sources are not reused. + through `USDGEO_CACHE_ROOT`. Reuse requires either a stable local filesystem + identity or a `Stable` resolver identity; `Unstable` and `Unavailable` + identity fails closed and reads from the source. Because only the conversion + tool publishes entries and it accepts `.las` and `.laz` local paths, a COPC + read has nothing to reuse outside a test that commits an entry itself. - No HTTP client, cloud SDK, authentication flow, retry policy, or raw byte-range cache exists here. Resolver-backed reads consume whatever the active `ArResolver` provides, and no resolver implementation is a build-time diff --git a/docs/reference/RESOLVER_BASELINE.md b/docs/reference/RESOLVER_BASELINE.md new file mode 100644 index 0000000..4a6a26c --- /dev/null +++ b/docs/reference/RESOLVER_BASELINE.md @@ -0,0 +1,131 @@ +# Resolver-backed read baseline + +Recorded Tier 2 numbers for reading a COPC asset through an external OpenUSD +resolver. The contract these measure is +[RESOLVER_SOURCE.md](../architecture/RESOLVER_SOURCE.md); Tier 1, the +repository-local gate that runs with no external resolver, is a different thing +and lives in the CI matrix. + +Last recorded: 2026-08-23, for `v0.10.0`. + +The machine-readable record is +[resolver-tier2-record.json](resolver-tier2-record.json). This file explains it; +the JSON is the evidence. + +## What produced these numbers + +| Component | Version | Role | +| --- | --- | --- | +| `usd-pointcloud-plugins` | `v0.10.0` | `pointcloud-copc` FileFormat, generated-cache decisions | +| [`usd-http-resolver`](https://github.com/animu-sphere/usd-http-resolver) | `v0.4.0` | `ArResolver` for `http`/`https`, range reads, identity through `ArAssetInfo` | +| `tools/tier2_fixture_server.py` | in-tree | loopback origin that honours `Range` and logs every request | +| Autzen classified COPC | PDAL `data` distribution | 81,123,042 bytes, 10,653,336 points, SHA-256 `db2d56cd…e25a27fa` | +| OpenStrata runtime | `cy2026` / `usd` (OpenUSD 26.08) | host | + +Neither repository is in the other's build graph. They compose at runtime +through `PXR_PLUGINPATH_NAME`, which is the whole of the integration. + +## Reproducing + +```powershell +# Fetch the fixture once; see docs/roadmap/streaming-and-tiling.md for provenance. +Invoke-WebRequest ` + -Uri https://s3.amazonaws.com/hobu-lidar/autzen-classified.copc.laz ` + -OutFile build/real-data-source/autzen-classified.copc.laz +Copy-Item build/real-data-source/autzen-classified.copc.laz build/tier2/autzen.copc + +Invoke-Expression (& ost env cy2026 --profile usd --shell powershell | Out-String) +python tools/tier2_resolver_integration.py ` + --fixture build/tier2/autzen.copc ` + --resolver-resources /plugins/http-resolver/plugin/resources/httpResolver ` + --copc-resources plugins/pointcloud-copc/plugin/resources/pointcloud-copc ` + --output build/tier2/record.json +``` + +The URL path must end in `.copc`, because OpenUSD selects the FileFormat by +extension. Each scenario runs in a fresh interpreter against a fresh origin, so +no in-process resolver state and no request log carries between rows. All +origins bind one port reserved for the whole run, so every row is served at one +identifier; the harness fails if that turns out not to hold, because three +revisions at three URLs would be three unrelated assets and would demonstrate +nothing about identity. + +## Recorded results + +`selectivity` is `bytes fetched / source size`. `identity` is the class this +repository derives from what the resolver published, and `codes` are the +generated-cache decision codes OpenUSD reported. + +| Scenario | Revision | Validator | Identity | Codes | Requests | Bytes fetched | Selectivity | Points | +| --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | +| full, local file | — | — | — | — | — | — | — | 10,653,336 | +| metadata only | A | strong | stable | — | 3 | 120,546 | 0.001486 | — | +| full read | A | strong | stable | COPC010 | 277 | 81,123,042 | 1.000000 | 10,653,336 | +| metadata only | B | strong | stable | — | 3 | 120,546 | 0.001486 | — | +| full read | B | strong | stable | COPC010 | 277 | 81,123,042 | 1.000000 | 10,653,336 | +| metadata only | W | weak | unstable | — | 3 | 120,546 | 0.001486 | — | +| full read | W | weak | unstable | COPC009 | 277 | 81,123,042 | 1.000000 | 10,653,336 | + +Revisions A, B, and W serve identical bytes at one identifier - +`http://127.0.0.1:/fixture.copc`, recorded in the JSON as `identifier` - +and differ only in the validator the origin publishes. That is deliberate: it +separates *what was resolved* from *which revision of it*, which is the +distinction the whole identity contract rests on. + +### Output equivalence + +Every row that authored points produced 10,653,336 points and the same SHA-256 +over the authored positions, `c6cb6109…76c1d2f6`, including the local-file row. +A resolver-backed read and a local read are the same authored asset. + +### Bounded read + +A metadata-only open costs three requests and 120,546 bytes: 0.15% of the asset. +The COPC header and its info VLR are what a metadata open needs, and that is +what crosses the wire. This is the number that justifies range access existing. + +### Full read + +A full read fetches the whole asset, in 277 requests of which 276 are ranges, +and selectivity is exactly 1.0. That is the correct answer, not a defect: a read +that authors every point needs every point. It is recorded because a range +reader that loses to a plain download on the full-read case has a coalescing +policy that the bounded number would be hiding. + +### Identity and the conservative fallback + +The strong-validator rows classify as `stable` and report `COPC010`, the +reuse-permitted category. The weak-validator row classifies as `unstable` and +reports `COPC009`, reuse disabled — and still authors identical output, because +a disabled cache changes what is reused, never what is read. + +That row is the fallback proven against a released external resolver rather than +against a test double: `usd-http-resolver` publishes a token in +`ArAssetInfo::version` only for a validator strong enough to prove two responses +are the same bytes, and this repository enables reuse only when a token is +present. Neither side negotiates; one value crosses the boundary. + +Revisions A and B carry different validation tokens for one identifier - the +recorded token digests are `38aee176…` and `172bc13c…` - so they derive +different generated-cache identities from the same generation inputs. Equal +identifiers never imply equal content, and this record is the demonstration. + +No row reports `COPC011`, the changed-identity category, and none can: that +category is emitted when a committed entry exists for a different validation +identity, and nothing in this harness publishes one. See the next section. + +## What this baseline does not measure + +- **Anything that requires a committed cache entry.** Generated entries are + published by `usd-pointcloud-convert`, which accepts `.las` and `.laz` local + inputs only, so no COPC read — local or resolver-backed — has an entry to hit + in a normal workflow. That rules out a generated-cache hit ratio, a + `generated-cache-hit`, and a `resolver-identity-changed` from this harness. + What it verifies is the decision that precedes them: which identity permits + reuse, and which diagnostic explains it. Reuse, invalidation on a changed + token, superseded-entry detection, incomplete-entry recovery, and + corrupted-entry recovery are covered by Tier 1 against committed entries. +- **Raw byte-range cache behavior.** That cache belongs to the resolver, and its + hit ratios are recorded in that repository's own baseline. +- **Wide-area network behavior.** The origin is loopback. These are protocol and + selectivity numbers, not latency numbers. diff --git a/docs/reference/resolver-tier2-record.json b/docs/reference/resolver-tier2-record.json new file mode 100644 index 0000000..c5b462c --- /dev/null +++ b/docs/reference/resolver-tier2-record.json @@ -0,0 +1,173 @@ +{ + "fixture": { + "name": "autzen.copc", + "sizeBytes": 81123042, + "sha256": "db2d56cdfa058bffccdc5d6019dae2fc9c6a551df10a5523c06c76a3e25a27fa" + }, + "scenarios": [ + { + "scenario": "full-local", + "target": "C:\\Users\\snkm\\AppData\\Local\\Temp\\tier2-resolver-dsotp5o0\\local.copc", + "opened": true, + "prims": [ + "PointCloud" + ], + "pointCount": 10653336, + "pointDigest": "c6cb61094db1b067a0bccccf54a1e284b9c035e4094d912de623b2fe76c1d2f6", + "elapsedSeconds": 17.9931, + "cacheRootConfigured": false, + "decisionCodes": [] + }, + { + "scenario": "metadata", + "target": "http://127.0.0.1:56980/fixture.copc", + "hasValidationToken": true, + "validationTokenDigest": "38aee17688176bc7", + "identityClass": "stable", + "opened": true, + "prims": [ + "PointCloud" + ], + "elapsedSeconds": 0.0309, + "cacheRootConfigured": true, + "decisionCodes": [], + "revision": "A", + "validatorStrength": "strong", + "origin": { + "requests": 3, + "rangeRequests": 2, + "bytesFetched": 120546, + "sourceBytes": 81123042, + "selectivity": 0.001486 + } + }, + { + "scenario": "full", + "target": "http://127.0.0.1:56980/fixture.copc", + "hasValidationToken": true, + "validationTokenDigest": "38aee17688176bc7", + "identityClass": "stable", + "opened": true, + "prims": [ + "PointCloud" + ], + "pointCount": 10653336, + "pointDigest": "c6cb61094db1b067a0bccccf54a1e284b9c035e4094d912de623b2fe76c1d2f6", + "elapsedSeconds": 18.7465, + "cacheRootConfigured": true, + "decisionCodes": [ + "COPC010" + ], + "revision": "A", + "validatorStrength": "strong", + "origin": { + "requests": 277, + "rangeRequests": 276, + "bytesFetched": 81123042, + "sourceBytes": 81123042, + "selectivity": 1.0 + } + }, + { + "scenario": "metadata", + "target": "http://127.0.0.1:56980/fixture.copc", + "hasValidationToken": true, + "validationTokenDigest": "172bc13c13af9aef", + "identityClass": "stable", + "opened": true, + "prims": [ + "PointCloud" + ], + "elapsedSeconds": 0.0297, + "cacheRootConfigured": true, + "decisionCodes": [], + "revision": "B", + "validatorStrength": "strong", + "origin": { + "requests": 3, + "rangeRequests": 2, + "bytesFetched": 120546, + "sourceBytes": 81123042, + "selectivity": 0.001486 + } + }, + { + "scenario": "full", + "target": "http://127.0.0.1:56980/fixture.copc", + "hasValidationToken": true, + "validationTokenDigest": "172bc13c13af9aef", + "identityClass": "stable", + "opened": true, + "prims": [ + "PointCloud" + ], + "pointCount": 10653336, + "pointDigest": "c6cb61094db1b067a0bccccf54a1e284b9c035e4094d912de623b2fe76c1d2f6", + "elapsedSeconds": 18.2973, + "cacheRootConfigured": true, + "decisionCodes": [ + "COPC010" + ], + "revision": "B", + "validatorStrength": "strong", + "origin": { + "requests": 277, + "rangeRequests": 276, + "bytesFetched": 81123042, + "sourceBytes": 81123042, + "selectivity": 1.0 + } + }, + { + "scenario": "metadata", + "target": "http://127.0.0.1:56980/fixture.copc", + "hasValidationToken": false, + "validationTokenDigest": "", + "identityClass": "unstable", + "opened": true, + "prims": [ + "PointCloud" + ], + "elapsedSeconds": 0.0289, + "cacheRootConfigured": true, + "decisionCodes": [], + "revision": "W", + "validatorStrength": "weak", + "origin": { + "requests": 3, + "rangeRequests": 2, + "bytesFetched": 120546, + "sourceBytes": 81123042, + "selectivity": 0.001486 + } + }, + { + "scenario": "full", + "target": "http://127.0.0.1:56980/fixture.copc", + "hasValidationToken": false, + "validationTokenDigest": "", + "identityClass": "unstable", + "opened": true, + "prims": [ + "PointCloud" + ], + "pointCount": 10653336, + "pointDigest": "c6cb61094db1b067a0bccccf54a1e284b9c035e4094d912de623b2fe76c1d2f6", + "elapsedSeconds": 18.0225, + "cacheRootConfigured": true, + "decisionCodes": [ + "COPC009" + ], + "revision": "W", + "validatorStrength": "weak", + "origin": { + "requests": 277, + "rangeRequests": 276, + "bytesFetched": 81123042, + "sourceBytes": 81123042, + "selectivity": 1.0 + } + } + ], + "identifier": "http://127.0.0.1:56980/fixture.copc" +} \ No newline at end of file diff --git a/docs/releases/README.md b/docs/releases/README.md index fb35a26..56e9d81 100644 --- a/docs/releases/README.md +++ b/docs/releases/README.md @@ -17,6 +17,7 @@ Release records are history and are not rewritten after publication. | v0.7.0 | 2026-08-13 | [v0.7.0.md](v0.7.0.md) — adaptive point-budget tiling, fixed-grid compatibility, and cross-format benchmarks | | v0.8.0 | 2026-08-14 | [v0.8.0.md](v0.8.0.md) — real-world fixed/adaptive baselines, I/O observability, and LAZ point-format-7 hardening | | v0.9.0 | 2026-08-15 | [v0.9.0.md](v0.9.0.md) — TilePlan convergence, COPC-native planning, and interactive host-responsiveness validation | +| v0.10.0 | 2026-08-23 | [v0.10.0.md](v0.10.0.md) — generated-cache decision diagnostics, a revision-aware cache layout, Windows workspace CTest runtime coverage, Tier 1 as a CI gate, and recorded external resolver interoperability | Prepare the record in the release commit immediately before creating its tag. The tag pins the source commit and the record pins the release scope; runtime diff --git a/docs/releases/v0.10.0.md b/docs/releases/v0.10.0.md new file mode 100644 index 0000000..d9ce9d3 --- /dev/null +++ b/docs/releases/v0.10.0.md @@ -0,0 +1,148 @@ +# OpenUSD Point Cloud Plugins v0.10.0 + +Release date: 2026-08-23 + +## Summary + +This feature release closes the resolver-backed source identity and external +resolver interoperability milestone. Generated-cache decisions are now +explained through a stable, transport-neutral vocabulary; the repository-local +resolver contract tests (Tier 1) run in CI on every host and both lanes; and +interoperability with a released external resolver is recorded with numbers +rather than described. + +## Breaking changes + +The generated cache entry layout moved from one directory below the cache root +to two: + +```text +# before +// +# after +/// +``` + +Cache entries are derived data, so a `v0.9.0` cache root is never looked up and +the first run after upgrading regenerates. Nothing else changes: no authored +output, file-format argument, USD attribute, or diagnostic code, and +`StableCacheKey`, `TryBuildLayout`, `Inspect`, `IsCacheHit`, and `Invalidate` +keep their signatures and meanings. Tooling that enumerated entries with a +single-level glob needs a second level; see +[MIGRATION.md](../compatibility/MIGRATION.md). + +Existing LAS, LAZ, COPC, and PLY format ids, file-format arguments, authored +stage shape, and fixed-grid tiling behavior remain compatible with v0.9.0. + +## Included + +- `usdgeo::cache::CacheDecision`, the stable vocabulary of seven + transport-neutral categories that explain a cache decision, with + `CacheDecisionName` as the machine-matchable form and fixed + `CacheDecisionMessage` constants that no transport specific can reach. +- A two-level generated cache entry layout that separates what the caller asked + for from everything read out of the source, making `resolver-identity-changed` + distinguishable from a source never generated before. `Descriptor` gained a + `sourceDerived` group for values a caller computed by scanning the source, + such as the conversion tool's tile-plan key. +- `HasSupersededIdentityEntry`, the probe that answers that question without + persisting an identifier or a validation token. +- Decision reporting through the shared authoring cache bridge, projected onto + four COPC codes, `COPC009` through `COPC012`, each message naming its exact + category. +- `kind: workspace` CI cells on Windows, macOS, and Linux for both the + pull-request and main lanes, which configure the repository root and run its + CTest suite. This is what makes the Tier 1 resolver gate a CI gate: a + per-plugin bundle cell never declares `USDGEO_BUILD_TESTS` and cannot compile + those tests. +- Windows CTest registrations derive the OpenUSD `lib` and `bin` directories + from the `usdGeom` imported target and prepend them together with the + configured Python runtime to `PATH`, so the authoring bridge, converter, and + FileFormat integration executables load their OpenUSD, TBB, and Python DLLs + without hiding host runtime DLLs in workspace CI. +- The PLY tiled-read integration fixture now writes payloads on its source + volume, preserving relative payload references when a Windows workspace and + the system temporary directory are on different volumes. +- `tools/tier2_fixture_server.py`, a loopback origin that honours `Range` and + logs every request, and `tools/tier2_resolver_integration.py`, the harness + that composes it with an external resolver and the COPC FileFormat. +- The recorded Tier 2 baseline in + [RESOLVER_BASELINE.md](../reference/RESOLVER_BASELINE.md). + +## Recorded interoperability + +Composed at runtime with +[`usd-http-resolver`](https://github.com/animu-sphere/usd-http-resolver) +`v0.4.0` over the 81,123,042-byte Autzen classified COPC. Neither repository is +in the other's build graph. + +| Scenario | Validator | Identity | Codes | Requests | Bytes fetched | Selectivity | Points | +| --- | --- | --- | --- | ---: | ---: | ---: | ---: | +| full, local file | — | — | — | — | — | — | 10,653,336 | +| metadata only | strong | stable | — | 3 | 120,546 | 0.001486 | — | +| full read | strong | stable | COPC010 | 277 | 81,123,042 | 1.000000 | 10,653,336 | +| metadata only | weak | unstable | — | 3 | 120,546 | 0.001486 | — | +| full read | weak | unstable | COPC009 | 277 | 81,123,042 | 1.000000 | 10,653,336 | + +Every row that authored points produced the same SHA-256 over the authored +positions, including the local-file row: a resolver-backed read and a local read +are the same authored asset. A second revision serving identical bytes under the +same identifier with a different validator derives a different generated-cache +identity, which is the property that equal identifiers never imply equal +content. + +The weak-validator row is the conservative fallback demonstrated against a real +resolver rather than a test double. `usd-http-resolver` publishes a token only +for a validator strong enough to prove two responses are the same bytes; this +project enables reuse only when a token is present; and the read still authors +identical output, because a disabled cache changes what is reused, never what is +read. + +## Secrets + +No credential, authorization header, signed URL, resolved identifier, or +validation token is persisted. Both cache path components are 64-bit hashes, +every decision message is a fixed constant, and the conversion conformance test +asserts that no manifest carries the source path or its validation token. + +## Requirements + +- CMake 3.23 or newer. +- A C++17 compiler. +- OpenUSD 26.08 for USD authoring, resolver, and plugin targets. +- OpenStrata 0.22.2 for the pinned workspace build and the `kind: workspace` CI + cells. + +## Known limitations + +- Nothing publishes a generated cache entry for a COPC source. + `usd-pointcloud-convert` accepts `.las` and `.laz` local inputs only, so a + COPC read — local or resolver-backed — has no entry to hit in a normal + workflow. Lookup, the reuse rules, and every decision diagnostic are complete + and covered; a measurable generated-cache hit ratio for a remote source waits + on COPC generation. +- `usd-pointcloud-convert` does not accept resolver-addressable identifiers. +- The Tier 2 origin is loopback. The recorded numbers are protocol and + selectivity numbers, not latency numbers. +- Raw byte-range caching and its hit ratios belong to the resolver and are + recorded in that repository. +- A full read fetches the whole asset. That is correct for a read that authors + every point, and it is recorded so a coalescing regression cannot hide behind + the bounded-read number. +- COPC writing and new public USD schemas remain deferred. + +## Licensing + +Original project code is licensed under Apache License 2.0. The LAZ and COPC +adapters incorporate `laz-perf 2.0.0`, which remains under LGPL-2.1. Release +source and binary products retain the existing notices and corresponding-source +materials. See [THIRD_PARTY_NOTICES.md](../../THIRD_PARTY_NOTICES.md), +[DISTRIBUTION.md](../guides/DISTRIBUTION.md), and [LICENSE](../../LICENSE). + +## Verification + +The release preparation validates all version declarations with +`python tools/check_release_metadata.py`. The local Windows `cy2026` / `usd` +Release build and the declared CTest suite are the final release checks; the +release workflow repeats the metadata, CI matrix, source, plugin, packaging, +and SHA-256 validation on the tagged commit. diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md index 4b22426..4592ca3 100644 --- a/docs/roadmap/README.md +++ b/docs/roadmap/README.md @@ -30,10 +30,11 @@ the conversion tool validate the shared point-cloud architecture. The next release sequence prioritizes infrastructure maturity over adding formats: the adaptive tiling shipped in `v0.7.0` is measured on real data before more tiling design is added, and the tile-plan representation is unified before another -format is connected to it. `v0.10.0` then closes the resolver-facing side — -transport-neutral source identity and safe generated-cache reuse for -resolver-provided sources — while transport itself stays in external resolver -implementations. The detailed rationale, scope, tests, and performance +format is connected to it. `v0.10.0` closed the resolver-facing side — +transport-neutral source identity, safe generated-cache reuse for +resolver-provided sources, and a stable vocabulary for every cache decision — +while transport itself stays in external resolver implementations, and recorded +that boundary working against a released one. The detailed rationale, scope, tests, and performance indicators are in the [infrastructure maturity roadmap](infrastructure-maturity.md); the boundary itself is fixed in the @@ -47,7 +48,7 @@ itself is fixed in the | `v0.7.0` | Adaptive tiling | Point-budget-aware payload density and memory use; released | | `v0.8.0` | Measurement and I/O observability | Real-world fixed-grid and adaptive baselines, and visible I/O amplification; released 2026-08-14 | | `v0.9.0` | TilePlan convergence and interactive validation | Sequential planning and COPC native hierarchy reach authoring through one representation, with a host-responsiveness baseline; released 2026-08-15 | -| `v0.10.0` | Resolver-backed source identity and external resolver interoperability | Generated-cache reuse for resolver-provided sources where identity is sufficient, with transport owned by the resolver | +| `v0.10.0` | Resolver-backed source identity and external resolver interoperability | Generated-cache reuse for resolver-provided sources where identity is sufficient, every decision explained through a stable category, Tier 1 as a CI gate, and recorded external interoperability; released 2026-08-23 | | Research | Runtime streaming | Host-driven partial loading investigated without complicating the conversion pipeline | | Later | Format expansion | E57 and other point-cloud adapters after infrastructure maturity | @@ -178,7 +179,7 @@ interrupted generated-cache entries. | 8 | Point-budget-aware adaptive tiling | Released in `v0.7.0` | Deterministic planning, tile statistics, fixed-grid compatibility, and a fixture-based cross-format baseline are implemented; broader real-world baselines remain | | 9 | Real-world measurement and I/O observability | Released in `v0.8.0` | Compare fixed-grid and adaptive on uneven real data, and make source, spool, and payload I/O visible | | 10 | TilePlan convergence and interactive validation | Released in `v0.9.0` | One plan representation for sequential planning and COPC native hierarchy, plus a reproducible host-responsiveness baseline | -| 11 | Resolver-backed source identity and external resolver interoperability | Planned for `v0.10.0` | Enable generated-cache reuse where a resolver supplies sufficient identity, keep transport out of the repository, and resolve the bundled test resolver's status | +| 11 | Resolver-backed source identity and external resolver interoperability | Released in `v0.10.0` | Generated-cache reuse where a resolver supplies sufficient identity, seven stable decision categories, transport kept out of the repository, the bundled test resolver isolated under `tests/`, and Tier 2 recorded against `usd-http-resolver` v0.4.0 | | 12 | E57 and other point-cloud formats | Deferred | Reuse `PointStream`, processing, authoring, and cache contracts | Runtime streaming is investigated in parallel with these phases and is not a @@ -203,7 +204,7 @@ maps onto the phases above. | W9 | Point-budget-aware adaptive tiling | 8 | Released in `v0.7.0` | | W10 | Real-world tiling baselines and I/O instrumentation | 9 | Released in `v0.8.0` | | W11 | `TilePlan` contract, adaptive migration, COPC native fast path, and interactive validation | 10 | Released in `v0.9.0` | -| W12 | Resolver source identity, range-cache ownership, external resolver interoperability, and remote baselines | 11 | Planned for `v0.10.0` | +| W12 | Resolver source identity, range-cache ownership, external resolver interoperability, and remote baselines | 11 | Released in `v0.10.0` | | W13 | Runtime streaming research | Parallel | Ongoing, no release gate | The completed workstreams established the shared point schema, streaming reader diff --git a/docs/roadmap/implementation-status.md b/docs/roadmap/implementation-status.md index 3b7598c..13cda5e 100644 --- a/docs/roadmap/implementation-status.md +++ b/docs/roadmap/implementation-status.md @@ -295,7 +295,7 @@ The ordered plan and acceptance priorities are in the - [x] Verify equivalent authored output from a sequential plan and a COPC-native plan describing the same partition -#### `v0.10.0` - resolver-backed source identity and external resolver interoperability (planned) +#### `v0.10.0` - resolver-backed source identity and external resolver interoperability (released 2026-08-23) The contract is recorded in the [resolver-backed source contract](../architecture/RESOLVER_SOURCE.md); the @@ -334,8 +334,11 @@ Phase 3 — generated cache reuse: - [x] Separate source byte-range caching from generated-USDC caching, with an explicit owner for each -The COPC FileFormat now reports disabled resolver-backed cache reuse through -the typed `COPC009` diagnostic while preserving the conservative fallback. +The COPC FileFormat reports every generated-cache decision through the typed +`COPC009`-`COPC012` diagnostics while preserving the conservative fallback. The +generated-cache layout separates a generation key from a source identity key, so +a changed validation token is reported as changed rather than as a source never +seen before. Phase 4 — repository boundary cleanup: @@ -351,20 +354,51 @@ Phase 4 — repository boundary cleanup: Phase 5 — diagnostics and secrets: -- [ ] Add stable diagnostics for identity unavailable, unstable, stable, and - changed, and for cache reuse disabled, hit, and invalidated -- [ ] Keep diagnostics free of transport specifics and token contents -- [ ] Verify no credentials, authorization headers, signed URLs, or tokens are - persisted into manifests or cache descriptors +- [x] Add stable diagnostics for identity unavailable, unstable, stable, and + changed, and for cache reuse disabled, hit, and invalidated. + `usdgeo::cache::CacheDecision` publishes the seven categories; COPC + projects them onto `COPC009`-`COPC012`, each message naming its category +- [x] Keep diagnostics free of transport specifics and token contents. Every + decision message is a fixed constant owned by `usdgeo::cache`, and a unit + test asserts none of them names a transport +- [x] Verify no credentials, authorization headers, signed URLs, or tokens are + persisted into manifests or cache descriptors. Both cache-path components + are 64-bit hashes, and the conversion conformance test asserts that no + manifest carries the source path or its validation token Phase 6 — validation and baselines: -- [ ] Pass Tier 1 repository-local resolver contract tests as the CI gate - without any external resolver repository -- [ ] Record Tier 2 integration against an external resolver, a local +- [x] Pass Tier 1 repository-local resolver contract tests as the CI gate + without any external resolver repository. `kind: workspace` cells + configure the repository root, where `USDGEO_BUILD_TESTS` defaults to + `ON`, on every host and both lanes +- [x] Record Tier 2 integration against an external resolver, a local reproducible HTTP server, and a COPC fixture, including local/remote - output equivalence -- [ ] Record remote hit ratios and `bytes fetched / source size` baselines + output equivalence. Recorded against `usd-http-resolver` `v0.4.0` and the + 81 MB Autzen COPC in the + [resolver read baseline](../reference/RESOLVER_BASELINE.md) +- [x] Record remote hit ratios and `bytes fetched / source size` baselines. A + metadata open costs 0.001486 of the asset in three requests; a full read + costs exactly 1.0 in 277 requests. A generated-cache hit ratio for COPC is + not measurable end to end, because `usd-pointcloud-convert` publishes + entries for `.las` and `.laz` local inputs only and no COPC read has an + entry to hit; the reuse decision itself is Tier 1 covered + +#### Open after `v0.10.0` + +- [ ] Declare an OST smoke fixture for the `pointcloud-copc` bundle. It is the + only product bundle without one, so its L3 `usdcat.read` and L4 + `python.stage_open` checks skip and its bundle cells verify discovery + only. + +- [ ] Publish generated cache entries for COPC inputs. Lookup is complete for + local and resolver-backed COPC, but `usd-pointcloud-convert` accepts + `.las` and `.laz` local paths only, so nothing populates an entry a COPC + read could hit. This is what a measurable generated-cache hit ratio for + remote sources is waiting on. +- [ ] Accept resolver-addressable identifiers in `usd-pointcloud-convert`, as + described in the + [resolver-backed source contract](../architecture/RESOLVER_SOURCE.md). #### Research - runtime streaming (no release gate) diff --git a/docs/roadmap/infrastructure-maturity.md b/docs/roadmap/infrastructure-maturity.md index eb7d553..eccf3f8 100644 --- a/docs/roadmap/infrastructure-maturity.md +++ b/docs/roadmap/infrastructure-maturity.md @@ -138,7 +138,7 @@ schema is deferred until the plain-attribute metadata contract is stable. | `v0.7.0` | Adaptive tiling | Predictable payload density and memory through point-budget planning | Released 2026-08-13 | | `v0.8.0` | Measurement and I/O observability | Real-world adaptive baselines and visible I/O amplification | Released 2026-08-14 | | `v0.9.0` | TilePlan convergence and interactive validation | One tile-plan representation for sequential planning and COPC native hierarchy, plus a host-responsiveness baseline | Released 2026-08-15 | -| `v0.10.0` | Resolver-backed source identity and external resolver interoperability | Safe generated-cache reuse for resolver-provided sources, with transport owned by the resolver | Planned | +| `v0.10.0` | Resolver-backed source identity and external resolver interoperability | Safe generated-cache reuse for resolver-provided sources, with transport owned by the resolver | Released 2026-08-23 | | Research | Runtime streaming | Evidence about host-driven partial loading, with no premature abstraction | Ongoing | | Later | Format expansion | E57 and other point-cloud formats through `PointStream` | Deferred | @@ -358,7 +358,7 @@ working-set pressure. It does not measure renderer frame latency or a host-specific input-to-present time; queued key messages are validated separately from the synchronous UI dispatch probe. -### `v0.10.0` - Resolver-Backed Source Identity +### `v0.10.0` - Resolver-Backed Source Identity (released 2026-08-23) Primary goal: make generated point-cloud assets safely reusable across resolver-backed sources without depending on any transport implementation. @@ -438,11 +438,13 @@ and is excluded from the product plugin and release matrices. #### Tests Tier 1 runs without any external resolver repository, using fake or -memory-backed test assets, and remains the required local gate; it is not yet -wired into the CI matrix, whose cells build plugin bundles individually rather -than the repository root: resolver-backed -random access, partial reads, short-read diagnostics, the three identity -states, miss-to-hit behavior, invalidation on token change, corruption +memory-backed test assets, and is the required gate on every host and both +lanes. `kind: workspace` cells configure the repository root, where +`USDGEO_BUILD_TESTS` defaults to `ON`, and run its CTest suite; a per-plugin +bundle cell cannot compile these tests, which is why the gate needed a second +cell kind rather than another bundle. Coverage: resolver-backed random access, +partial reads, short-read diagnostics, the three identity states, miss-to-hit +behavior, invalidation on token change, superseded-entry detection, corruption recovery, `TilePlan` compatibility inputs, and deterministic diagnostics. Tier 2 composes an external resolver, a local reproducible HTTP server, and a @@ -451,11 +453,15 @@ metadata and range reads, local/remote output equivalence, reuse under stable identity, and invalidation when validation metadata changes. [`usd-http-resolver`](https://github.com/animu-sphere/usd-http-resolver) is the -first external implementation. Its `v0.2.0` release provides the HTTP backend, -OpenUSD resolver bundle, stable `ArAssetInfo` identity, and OpenStrata build and -test workflow. Tier 1 remains the dependency-free gate here; Tier 2 can now be -composed against the released resolver and recorded independently of this -repository's build graph. +first external implementation, and `v0.10.0` is recorded against its `v0.4.0` +release. The run is reproducible from `tools/tier2_fixture_server.py` and +`tools/tier2_resolver_integration.py`, and the numbers are in the +[resolver read baseline](../reference/RESOLVER_BASELINE.md): a metadata open of +the 81 MB Autzen COPC costs three requests and 0.001486 of the asset, a full +read costs 277 requests and exactly 1.0, and local, remote, and +second-revision reads author the same 10,653,336 points under the same digest. +A weak validator classifies as `Unstable`, disables reuse, and still authors +identical output. #### Diagnostics @@ -473,11 +479,20 @@ protocols, signed URLs, S3 / Azure / GCS SDK integration, raw remote byte caching, range-cache eviction, connection pooling, COPC writing, E57, public custom point-cloud USD schemas, and renderer-controlled runtime streaming. -Exit gate: a documented identity contract for resolver-provided sources, reuse -enabled exactly where identity is sufficient, Tier 1 passing without an -external resolver, the bundled test resolver isolated under `tests/`, and -recorded remote baselines including `bytes fetched / source size`. Before the -release is tagged, Tier 2 must be recorded against a released external resolver. +Exit gate, met at the tag: a documented identity contract for +resolver-provided sources, reuse enabled exactly where identity is sufficient, +Tier 1 passing in CI without an external resolver, the bundled test resolver +isolated under `tests/`, and recorded remote baselines including +`bytes fetched / source size` against the released `usd-http-resolver` +`v0.4.0`. + +One thing the gate asked for is recorded as absent rather than as met: a +generated-cache *hit ratio* for a remote source. Entries are published only by +`usd-pointcloud-convert`, which accepts `.las` and `.laz` local inputs, so no +COPC read - local or resolver-backed - has an entry to hit in a normal +workflow. The lookup side, the reuse rules, and every decision diagnostic are +complete and covered; publishing COPC entries is the follow-up that makes the +ratio measurable. ### Research - Runtime Streaming diff --git a/libs/usd-geo-cache/README.md b/libs/usd-geo-cache/README.md index 0a3f74a..7e4d3dc 100644 --- a/libs/usd-geo-cache/README.md +++ b/libs/usd-geo-cache/README.md @@ -7,7 +7,41 @@ and downsampling settings. The descriptor is normalized through `usdGeoCore::StableCacheKey` so equivalent argument order and whitespace do not create different cache entries. -Each descriptor maps to one entry directory with reserved paths for +Each descriptor maps to one entry directory two levels below the cache root: + +```text +/// +``` + +Both components are the 16-hex-character `StableCacheKey` hash, and the split +follows one rule: caller intent chooses the directory, and everything read out +of the source chooses the entry inside it. + +| Half | `Descriptor` fields | +| --- | --- | +| generation key | `source.identifier`, `pluginVersion`, `parserVersion`, `openUsdVersion`, `attributes`, `tileAndLod`, `downsampling` | +| source identity key | `source.sizeBytes`, `source.modifiedTime`, `source.validationToken`, `coordinateTransform`, `sourceDerived` | + +`coordinateTransform` is in the second half because it is resolved from the +source header: its local origin is the source bounding box and its CRS may be an +embedded record. `sourceDerived` is for anything else a caller computed by +reading the source, such as the conversion tool's tile-plan key. Planner +identity and version are caller intent and stay in `tileAndLod`. + +Putting a source-derived value in the first half is a defect rather than a +preference: a revised source would land in an unrelated generation directory, +and `HasSupersededIdentityEntry` could no longer see that it superseded +anything. A unit test holds both directions - every revision-varying field keeps +the generation directory, and every caller-intent field changes it. + +The split is not cosmetic. It puts every revision of one source in one +generation directory, so `HasSupersededIdentityEntry` can tell a changed +validation token from a source that was never generated before, which is what +the `resolver-identity-changed` decision reports. It also keeps identifiers and +tokens off the filesystem: a signed URL cannot be read back out of a cache +root, because both components are hashes. + +An entry directory has reserved paths for `root.usdc`, a `cache.manifest`, and a `payloads/` directory. `TilePayloadPath` uses the same stable tile naming shape as payload-backed authoring. Cache entries are deletable derived data. The manifest is the generation commit @@ -18,7 +52,9 @@ interrupted generation is reported as `Incomplete` and is not reusable. for an invalid layout. `IsCacheHit` remains the compatibility boolean wrapper around this status contract. `Invalidate` recomputes the descriptor entry below the supplied cache root and never accepts an arbitrary layout path, so it -cannot delete an unrelated sibling directory. +cannot delete an unrelated sibling directory. It removes the generation +directory too once its last identity entry is gone, so an invalidated root does +not accumulate empty parents. `LookupStatusName` exposes stable machine-readable status names, and the cache-library `GetLookupStatistics` snapshot counts every `Inspect` call by status. Updates, snapshots, and resets are serialized so the counters remain @@ -33,6 +69,27 @@ payload, or a payload outside the entry is treated as a corrupt entry and invalidated; failures materializing valid cached payloads into a caller-owned directory do not invalidate the cache entry. +## Cache Decisions + +`CacheDecision` is the stable, transport-neutral vocabulary that explains what a +lookup did. `CacheDecisionName` is the string form a consumer matches on, and +`CacheDecisionMessage` is a fixed human-readable constant - fixed so that no +transport specific and no token content can reach a diagnostic by accident: + +| Name | Meaning | +| --- | --- | +| `resolver-identity-unavailable` | the resolver exposed no usable identity metadata | +| `resolver-identity-unstable` | the source is readable but its freshness is not guaranteed | +| `resolver-identity-stable` | identity is sufficient for reuse | +| `resolver-identity-changed` | an entry exists for a different validation identity | +| `generated-cache-reuse-disabled` | reuse was not attempted | +| `generated-cache-hit` | a committed entry was reused | +| `generated-cache-invalidated` | an entry did not validate and was removed | + +`IdentityDecision` maps a `ResolverIdentityStability` onto the matching identity +category. Names are stable once published; messages are for humans and may +change between releases. + ## Compatibility and Invalidation An entry is compatible only when its descriptor produces the same stable key. @@ -77,8 +134,8 @@ written into cache descriptors, manifests, or diagnostics. Resolver-provided identity is classified before it is trusted — stable, unstable, or unavailable — and generated-cache reuse is enabled only for the -stable case. `ClassifyResolverIdentity` and -`TryBuildResolverSourceIdentity` provide this transport-neutral Phase 1 -contract without depending on OpenUSD or a resolver implementation. The -OpenUSD-facing adapter remains v0.10.0 work; see the +stable case. `ClassifyResolverIdentity` and `TryBuildResolverSourceIdentity` +provide this transport-neutral contract without depending on OpenUSD or a +resolver implementation. The OpenUSD-facing adapter that feeds them lives in +`usdPointCloudAuthoring`, so this module never sees an `ArResolver`; see the [resolver-backed source contract](../../docs/architecture/RESOLVER_SOURCE.md). \ No newline at end of file diff --git a/libs/usd-geo-cache/include/usdgeo/cache/Cache.h b/libs/usd-geo-cache/include/usdgeo/cache/Cache.h index 9d87e0d..6b27bb3 100644 --- a/libs/usd-geo-cache/include/usdgeo/cache/Cache.h +++ b/libs/usd-geo-cache/include/usdgeo/cache/Cache.h @@ -18,6 +18,23 @@ enum class ResolverIdentityStability { const char* ResolverIdentityStabilityName( ResolverIdentityStability stability) noexcept; +// Stable, transport-neutral categories used to explain a cache decision. +// Names and meanings are published; messages are for humans and may change. +// No value ever carries transport specifics or validation-token contents. +enum class CacheDecision { + IdentityUnavailable, + IdentityUnstable, + IdentityStable, + IdentityChanged, + ReuseDisabled, + Hit, + Invalidated, +}; + +const char* CacheDecisionName(CacheDecision decision) noexcept; +const char* CacheDecisionMessage(CacheDecision decision) noexcept; +CacheDecision IdentityDecision(ResolverIdentityStability stability) noexcept; + struct ResolverAssetIdentity { std::string resolvedIdentifier; std::uintmax_t sizeBytes = 0; @@ -35,15 +52,37 @@ struct SourceIdentity { bool IsValid() const noexcept; }; +// A descriptor has two halves, and which half a value belongs in decides +// whether two reads of one source are recognizable as revisions of each other. +// +// caller intent what was asked for, independent of the bytes: +// versions, attribute selection, tiling and LOD arguments, +// downsampling. These choose the generation directory. +// source-derived what was read out of the source: its revision metadata, +// the georeference resolved from its header, and any plan +// computed by scanning it. These choose the entry inside +// that directory. +// +// Putting a source-derived value in the caller-intent half is a defect, not a +// preference: a revised source would land in an unrelated generation directory, +// and `HasSupersededIdentityEntry` could no longer see that it superseded +// anything. struct Descriptor { SourceIdentity source; std::string pluginVersion; std::string parserVersion; std::string openUsdVersion; + // Source-derived: the georeference resolved from the source header, whose + // local origin is the source bounding box and whose CRS may come from an + // embedded record. usdgeo::CacheArguments coordinateTransform; usdgeo::CacheArguments attributes; usdgeo::CacheArguments tileAndLod; usdgeo::CacheArguments downsampling; + // Source-derived: anything else a caller computed by reading the source, + // such as a tile-plan key produced by scanning it. Planner identity and + // version are caller intent and belong in `tileAndLod`. + usdgeo::CacheArguments sourceDerived; bool IsValid() const noexcept; }; @@ -107,6 +146,12 @@ std::filesystem::path TilePayloadPath(const Layout& layout, int lodLevel); LookupResult Inspect(const Layout& layout) noexcept; + +// True when the generation directory that owns this entry already holds a +// committed entry for a different source validation identity. It is how a +// changed validation token is distinguished from a source never generated +// before, without persisting the token or the resolved identifier. +bool HasSupersededIdentityEntry(const Layout& layout) noexcept; bool IsCacheHit(const Layout& layout) noexcept; LookupStatistics GetLookupStatistics() noexcept; void ResetLookupStatistics() noexcept; diff --git a/libs/usd-geo-cache/src/Cache.cpp b/libs/usd-geo-cache/src/Cache.cpp index beb8993..c42c1df 100644 --- a/libs/usd-geo-cache/src/Cache.cpp +++ b/libs/usd-geo-cache/src/Cache.cpp @@ -64,6 +64,27 @@ std::string TileName(const usdgeo::TileId& tile) { "_" + AxisName(tile.y) + "_" + AxisName(tile.z); } +constexpr const char* kRootLayerName = "root.usdc"; +constexpr const char* kManifestName = "cache.manifest"; +constexpr const char* kPayloadDirectoryName = "payloads"; + +// Layout keys are the 16 lowercase hex characters `usdgeo::StableCacheKey` +// emits. Anything else in a generation directory - a converter's temporary +// entry, an unrelated file - is not a committed sibling entry. +bool IsLayoutKeyName(const std::string& name) noexcept { + if (name.size() != 16) { + return false; + } + for (const char character : name) { + const bool digit = character >= '0' && character <= '9'; + const bool lower = character >= 'a' && character <= 'f'; + if (!digit && !lower) { + return false; + } + } + return true; +} + struct MarkerStatus { bool exists = false; bool regular = false; @@ -115,6 +136,67 @@ const char* ResolverIdentityStabilityName( return "unavailable"; } +const char* CacheDecisionName(CacheDecision decision) noexcept { + switch (decision) { + case CacheDecision::IdentityUnavailable: + return "resolver-identity-unavailable"; + case CacheDecision::IdentityUnstable: + return "resolver-identity-unstable"; + case CacheDecision::IdentityStable: + return "resolver-identity-stable"; + case CacheDecision::IdentityChanged: + return "resolver-identity-changed"; + case CacheDecision::ReuseDisabled: + return "generated-cache-reuse-disabled"; + case CacheDecision::Hit: + return "generated-cache-hit"; + case CacheDecision::Invalidated: + return "generated-cache-invalidated"; + } + return "generated-cache-reuse-disabled"; +} + +const char* CacheDecisionMessage(CacheDecision decision) noexcept { + switch (decision) { + case CacheDecision::IdentityUnavailable: + return "Source identity unavailable: the active resolver exposed no " + "usable identity metadata."; + case CacheDecision::IdentityUnstable: + return "Source identity unstable: the active resolver identified the " + "source but could not guarantee its freshness."; + case CacheDecision::IdentityStable: + return "Source identity stable: generated cache reuse is permitted."; + case CacheDecision::IdentityChanged: + return "Source identity changed: a generated entry exists for a " + "different source validation identity, so output is " + "regenerated."; + case CacheDecision::ReuseDisabled: + return "Generated cache reuse disabled: the active resolver did not " + "provide a stable source validation identity."; + case CacheDecision::Hit: + return "Generated cache hit: the committed entry matched and was " + "reused."; + case CacheDecision::Invalidated: + return "Generated cache invalidated: the committed entry did not " + "validate and was removed."; + } + return "Generated cache reuse disabled: the active resolver did not " + "provide a stable source validation identity."; +} + +CacheDecision IdentityDecision( + ResolverIdentityStability stability) noexcept { + switch (stability) { + case ResolverIdentityStability::Stable: + return CacheDecision::IdentityStable; + case ResolverIdentityStability::Unstable: + return CacheDecision::IdentityUnstable; + case ResolverIdentityStability::Unavailable: + return CacheDecision::IdentityUnavailable; + } + return CacheDecision::IdentityUnavailable; +} + double LookupStatistics::HitRatio() const noexcept { return lookups == 0 ? 0.0 : static_cast(hits) / @@ -132,7 +214,8 @@ bool Descriptor::IsValid() const noexcept { !HasArgumentsWithEmptyNames(coordinateTransform) && !HasArgumentsWithEmptyNames(attributes) && !HasArgumentsWithEmptyNames(tileAndLod) && - !HasArgumentsWithEmptyNames(downsampling); + !HasArgumentsWithEmptyNames(downsampling) && + !HasArgumentsWithEmptyNames(sourceDerived); } bool Layout::IsValid() const noexcept { @@ -242,32 +325,62 @@ bool TryBuildResolverSourceIdentity( return true; } -usdgeo::CacheArguments MakeCacheArguments(const Descriptor& descriptor) { +namespace { + +const std::string& SourceValidation(const Descriptor& descriptor) { + return descriptor.source.validationToken.empty() + ? descriptor.source.contentIdentity + : descriptor.source.validationToken; +} + +void AppendPrefixed(usdgeo::CacheArguments& arguments, + const char* prefix, + const usdgeo::CacheArguments& values) { + for (const auto& [name, value] : values) { + arguments.emplace_back(std::string(prefix) + "." + name, value); + } +} + +// The caller-intent half: what was asked for, independent of what the source +// turned out to contain. Nothing here may vary between two revisions of one +// source, or those revisions stop being siblings and a changed validation +// identity becomes indistinguishable from a source never seen before. +usdgeo::CacheArguments MakeGenerationArguments(const Descriptor& descriptor) { const auto& sourceIdentifier = descriptor.source.identifier.empty() ? descriptor.source.canonicalPath : descriptor.source.identifier; - const auto& sourceValidation = descriptor.source.validationToken.empty() - ? descriptor.source.contentIdentity - : descriptor.source.validationToken; usdgeo::CacheArguments arguments{ {"source.identifier", sourceIdentifier}, - {"source.size", std::to_string(descriptor.source.sizeBytes)}, - {"source.modified", std::to_string(descriptor.source.modifiedTime)}, - {"source.validation", sourceValidation}, {"plugin.version", descriptor.pluginVersion}, {"parser.version", descriptor.parserVersion}, {"openusd.version", descriptor.openUsdVersion}}; + AppendPrefixed(arguments, "attributes", descriptor.attributes); + AppendPrefixed(arguments, "tile-lod", descriptor.tileAndLod); + AppendPrefixed(arguments, "downsampling", descriptor.downsampling); + return arguments; +} - const auto append = [&arguments](const char* prefix, - const usdgeo::CacheArguments& values) { - for (const auto& [name, value] : values) { - arguments.emplace_back(std::string(prefix) + "." + name, value); - } - }; - append("transform", descriptor.coordinateTransform); - append("attributes", descriptor.attributes); - append("tile-lod", descriptor.tileAndLod); - append("downsampling", descriptor.downsampling); +// The source-derived half: the revision metadata the filesystem or a resolver +// reports, the georeference resolved out of the source header - its local +// origin is the source bounding box, and its CRS may be an embedded record - +// and anything else a caller computed by scanning the source. +usdgeo::CacheArguments MakeIdentityArguments(const Descriptor& descriptor) { + usdgeo::CacheArguments arguments{ + {"source.size", std::to_string(descriptor.source.sizeBytes)}, + {"source.modified", std::to_string(descriptor.source.modifiedTime)}, + {"source.validation", SourceValidation(descriptor)}}; + AppendPrefixed(arguments, "transform", descriptor.coordinateTransform); + AppendPrefixed(arguments, "source-derived", descriptor.sourceDerived); + return arguments; +} + +} // namespace + +usdgeo::CacheArguments MakeCacheArguments(const Descriptor& descriptor) { + auto arguments = MakeGenerationArguments(descriptor); + for (auto& [name, value] : MakeIdentityArguments(descriptor)) { + arguments.emplace_back(name, value); + } return arguments; } @@ -286,15 +399,24 @@ bool TryBuildLayout(const std::filesystem::path& cacheRoot, return false; } - const auto key = StableCacheKey(descriptor); - if (key.empty()) { + // Two levels: the generation inputs choose the directory, the source + // validation identity chooses the entry inside it. Equal generation inputs + // therefore collect every revision of the same source side by side, which + // is what lets a changed validation token be reported as changed instead of + // as never seen. Neither level stores the identifier or the token itself; + // both are hashes. + const auto generationKey = + usdgeo::StableCacheKey(MakeGenerationArguments(descriptor)); + const auto identityKey = + usdgeo::StableCacheKey(MakeIdentityArguments(descriptor)); + if (generationKey.empty() || identityKey.empty()) { return false; } - layout.entryDirectory = cacheRoot / key; - layout.rootLayer = layout.entryDirectory / "root.usdc"; - layout.manifest = layout.entryDirectory / "cache.manifest"; - layout.payloadDirectory = layout.entryDirectory / "payloads"; + layout.entryDirectory = cacheRoot / generationKey / identityKey; + layout.rootLayer = layout.entryDirectory / kRootLayerName; + layout.manifest = layout.entryDirectory / kManifestName; + layout.payloadDirectory = layout.entryDirectory / kPayloadDirectoryName; return true; } @@ -329,6 +451,40 @@ LookupResult Inspect(const Layout& layout) noexcept { return {status}; } +bool HasSupersededIdentityEntry(const Layout& layout) noexcept { + if (!layout.IsValid()) { + return false; + } + const auto generationDirectory = layout.entryDirectory.parent_path(); + if (generationDirectory.empty()) { + return false; + } + std::error_code error; + std::filesystem::directory_iterator iterator(generationDirectory, error); + const std::filesystem::directory_iterator end; + if (error) { + return false; + } + const auto entryName = layout.entryDirectory.filename().string(); + for (; iterator != end; iterator.increment(error)) { + if (error) { + return false; + } + const auto& path = iterator->path(); + const auto name = path.filename().string(); + if (name == entryName || !IsLayoutKeyName(name)) { + continue; + } + const auto root = InspectMarker(path / kRootLayerName); + const auto manifest = InspectMarker(path / kManifestName); + if (root.exists && root.regular && manifest.exists && + manifest.regular) { + return true; + } + } + return false; +} + bool IsCacheHit(const Layout& layout) noexcept { return Inspect(layout).IsHit(); } @@ -356,7 +512,14 @@ bool Invalidate(const std::filesystem::path& cacheRoot, } std::error_code error; std::filesystem::remove_all(layout.entryDirectory, error); - return !error; + if (error) { + return false; + } + // Drop the generation directory once its last identity entry is gone, so + // an invalidated cache root does not accumulate empty parents. + std::error_code cleanupError; + std::filesystem::remove(layout.entryDirectory.parent_path(), cleanupError); + return true; } } // namespace usdgeo::cache \ No newline at end of file diff --git a/libs/usd-geo-cache/tests/test_cache.cpp b/libs/usd-geo-cache/tests/test_cache.cpp index af16c6d..590ac27 100644 --- a/libs/usd-geo-cache/tests/test_cache.cpp +++ b/libs/usd-geo-cache/tests/test_cache.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace { @@ -153,6 +154,185 @@ void TestResolverSourceIdentity() { Check(!stableIdentity.IsValid()); } +void TestCacheDecisionVocabulary() { + using usdgeo::cache::CacheDecision; + const std::vector> expected{ + {CacheDecision::IdentityUnavailable, "resolver-identity-unavailable"}, + {CacheDecision::IdentityUnstable, "resolver-identity-unstable"}, + {CacheDecision::IdentityStable, "resolver-identity-stable"}, + {CacheDecision::IdentityChanged, "resolver-identity-changed"}, + {CacheDecision::ReuseDisabled, "generated-cache-reuse-disabled"}, + {CacheDecision::Hit, "generated-cache-hit"}, + {CacheDecision::Invalidated, "generated-cache-invalidated"}}; + for (const auto& [decision, name] : expected) { + Check(usdgeo::cache::CacheDecisionName(decision) == name, + "cache decision name"); + const std::string message = + usdgeo::cache::CacheDecisionMessage(decision); + Check(!message.empty(), "cache decision message"); + // Transport specifics never enter a decision message, and neither do + // token or identifier contents: the strings are fixed constants. + for (const char* forbidden : + {"http", "HTTP", "ETag", "etag", "url", "URL", "Authorization", + "token:", "s3", "S3"}) { + Check(message.find(forbidden) == std::string::npos, + "cache decision message leaked transport detail"); + } + } + + Check(usdgeo::cache::IdentityDecision( + usdgeo::cache::ResolverIdentityStability::Stable) == + CacheDecision::IdentityStable); + Check(usdgeo::cache::IdentityDecision( + usdgeo::cache::ResolverIdentityStability::Unstable) == + CacheDecision::IdentityUnstable); + Check(usdgeo::cache::IdentityDecision( + usdgeo::cache::ResolverIdentityStability::Unavailable) == + CacheDecision::IdentityUnavailable); +} + +// A cache root must never become a place to read back what was resolved. Both +// path components are hashes, so neither a signed URL nor a validation token +// survives into the layout a lookup writes. +void TestLayoutCarriesNoSecrets() { + const auto root = std::filesystem::temp_directory_path() / + "usdgeo-cache-secret-check"; + auto descriptor = MakeDescriptor(); + descriptor.source = { + "https://example.org/data.copc?X-Amz-Signature=deadbeefcafe", + 4096, + 99, + "opaque-validation-9f1c"}; + usdgeo::cache::Layout layout; + Check(usdgeo::cache::TryBuildLayout(root, descriptor, layout)); + + const auto rendered = layout.entryDirectory.generic_string() + "|" + + layout.rootLayer.generic_string() + "|" + + layout.manifest.generic_string() + "|" + + layout.payloadDirectory.generic_string(); + for (const char* secret : {"X-Amz-Signature", "deadbeefcafe", + "example.org", "https", "opaque-validation", + "9f1c"}) { + Check(rendered.find(secret) == std::string::npos, + "cache layout leaked source identity material"); + } + + const auto generationKey = + layout.entryDirectory.parent_path().filename().string(); + const auto identityKey = layout.entryDirectory.filename().string(); + Check(generationKey.size() == 16 && identityKey.size() == 16, + "cache layout keys are 64-bit hashes"); +} + +// Changing the validation token must move the entry inside the same generation +// directory, which is what makes `resolver-identity-changed` distinguishable +// from a source that was never generated before. +void TestSupersededIdentityEntry() { + const auto uniqueSuffix = + std::chrono::steady_clock::now().time_since_epoch().count(); + const auto root = std::filesystem::temp_directory_path() / + ("usdgeo-cache-identity-" + std::to_string(uniqueSuffix)); + + const auto first = MakeDescriptor(); + usdgeo::cache::Layout firstLayout; + Check(usdgeo::cache::TryBuildLayout(root, first, firstLayout)); + + // Everything a revision of the same source can change must move the entry + // without moving the generation directory. The georeference is on this list + // because its local origin is the source bounding box and its CRS may be an + // embedded record, and the tile plan is because it is computed by scanning + // the source - both vary with content, not with what the caller asked for. + const auto revise = [](usdgeo::cache::Descriptor descriptor, + int which) { + switch (which) { + case 0: + descriptor.source.validationToken = "sha256:def"; + break; + case 1: + descriptor.source.sizeBytes += 17; + break; + case 2: + descriptor.source.modifiedTime += 1; + break; + case 3: + descriptor.coordinateTransform = {{"origin", "101,200,0"}}; + break; + default: + descriptor.sourceDerived = {{"tile.plan", "0123456789abcdef"}}; + break; + } + return descriptor; + }; + for (int which = 0; which != 5; ++which) { + usdgeo::cache::Layout revised; + Check(usdgeo::cache::TryBuildLayout(root, revise(first, which), + revised)); + Check(revised.entryDirectory != firstLayout.entryDirectory, + "a revised source reused the entry it supersedes"); + Check(revised.entryDirectory.parent_path() == + firstLayout.entryDirectory.parent_path(), + "a revised source left the generation directory"); + } + + // What the caller asked for is the other half: changing it must move the + // generation directory, so unrelated requests are never mistaken for + // revisions of each other. + const auto reask = [](usdgeo::cache::Descriptor descriptor, int which) { + switch (which) { + case 0: + descriptor.attributes = {{"selection", "xyz"}}; + break; + case 1: + descriptor.tileAndLod = {{"tileSize", "128"}}; + break; + case 2: + descriptor.downsampling = {{"algorithm", "fixed-stride"}, + {"version", "2"}}; + break; + default: + descriptor.pluginVersion = "0.2.2"; + break; + } + return descriptor; + }; + for (int which = 0; which != 4; ++which) { + usdgeo::cache::Layout other; + Check(usdgeo::cache::TryBuildLayout(root, reask(first, which), other)); + Check(other.entryDirectory.parent_path() != + firstLayout.entryDirectory.parent_path(), + "different generation inputs shared a generation directory"); + } + + const auto second = revise(first, 0); + usdgeo::cache::Layout secondLayout; + Check(usdgeo::cache::TryBuildLayout(root, second, secondLayout)); + + Check(!usdgeo::cache::HasSupersededIdentityEntry(secondLayout), + "empty cache reported a superseded entry"); + std::filesystem::create_directories(firstLayout.payloadDirectory); + std::ofstream(firstLayout.rootLayer) << "cache"; + + Check(!usdgeo::cache::HasSupersededIdentityEntry(secondLayout), + "uncommitted entry reported as superseded"); + std::ofstream(firstLayout.manifest) << "committed"; + Check(usdgeo::cache::HasSupersededIdentityEntry(secondLayout), + "committed sibling entry not reported as superseded"); + Check(!usdgeo::cache::HasSupersededIdentityEntry(firstLayout), + "an entry reported itself as superseded"); + + // A converter's temporary entry is a sibling directory but not a key, so + // it must never be mistaken for a committed revision. + const auto temporary = std::filesystem::path( + secondLayout.entryDirectory.string() + ".tmp-1-0"); + std::filesystem::create_directories(temporary); + std::ofstream(temporary / "root.usdc") << "cache"; + std::ofstream(temporary / "cache.manifest") << "committed"; + Check(!usdgeo::cache::HasSupersededIdentityEntry(firstLayout), + "temporary entry reported as a committed revision"); + + std::filesystem::remove_all(root); +} + void TestLayoutAndInvalidation() { const auto uniqueSuffix = std::chrono::steady_clock::now().time_since_epoch().count(); @@ -194,6 +374,8 @@ void TestLayoutAndInvalidation() { std::ofstream(unrelatedDirectory / "source.las") << "source"; Check(usdgeo::cache::Invalidate(root, MakeDescriptor())); Check(!std::filesystem::exists(layout.entryDirectory)); + Check(!std::filesystem::exists(layout.entryDirectory.parent_path()), + "invalidation left an empty generation directory behind"); Check(std::filesystem::exists(unrelatedDirectory / "source.las")); } @@ -291,6 +473,9 @@ int main() { TestLocalSourceIdentity(); TestStableDescriptorKey(); TestResolverSourceIdentity(); + TestCacheDecisionVocabulary(); + TestLayoutCarriesNoSecrets(); + TestSupersededIdentityEntry(); TestLayoutAndInvalidation(); TestLookupStatistics(); TestConcurrentLookupStatistics(); diff --git a/libs/usd-pointcloud-authoring/include/usdgeo/PointCloudCache.h b/libs/usd-pointcloud-authoring/include/usdgeo/PointCloudCache.h index 667f0aa..ee4d49a 100644 --- a/libs/usd-pointcloud-authoring/include/usdgeo/PointCloudCache.h +++ b/libs/usd-pointcloud-authoring/include/usdgeo/PointCloudCache.h @@ -33,6 +33,12 @@ bool TryBuildResolverSourceIdentity( cache::ResolverIdentityStability& stability, std::string& errorMessage); +// `decision`, when given, receives the stable category that explains what the +// lookup did: `Hit` when a committed entry was reused, `Invalidated` when one +// was rejected and removed, `IdentityChanged` when the same generation inputs +// already hold an entry for a different source validation identity. A plain +// first-time miss leaves it untouched, so a caller that has already classified +// the source identity keeps that classification. bool TryLoadPointCloudCache( pxr::SdfLayer* layer, const std::filesystem::path& sourcePath, @@ -40,7 +46,8 @@ bool TryLoadPointCloudCache( const usdpointcloud::PointReadRequest& request, const std::string& parserVersion, bool& hit, - std::string& errorMessage); + std::string& errorMessage, + cache::CacheDecision* decision = nullptr); bool TryLoadPointCloudCache( pxr::SdfLayer* layer, @@ -50,6 +57,7 @@ bool TryLoadPointCloudCache( const usdpointcloud::PointReadRequest& request, const std::string& parserVersion, bool& hit, - std::string& errorMessage); + std::string& errorMessage, + cache::CacheDecision* decision = nullptr); } // namespace usdgeo diff --git a/libs/usd-pointcloud-authoring/src/PointCloudCache.cpp b/libs/usd-pointcloud-authoring/src/PointCloudCache.cpp index fe96bf7..71bfef8 100644 --- a/libs/usd-pointcloud-authoring/src/PointCloudCache.cpp +++ b/libs/usd-pointcloud-authoring/src/PointCloudCache.cpp @@ -352,7 +352,8 @@ bool TryLoadPointCloudCache( const usdpointcloud::PointReadRequest& request, const std::string& parserVersion, bool& hit, - std::string& errorMessage) { + std::string& errorMessage, + cache::CacheDecision* decision) { hit = false; if (!layer) { errorMessage = "cache lookup requires a writable layer"; @@ -369,7 +370,7 @@ bool TryLoadPointCloudCache( } return TryLoadPointCloudCache( layer, sourceIdentity, sourcePath.parent_path(), reference, request, - parserVersion, hit, errorMessage); + parserVersion, hit, errorMessage, decision); } bool TryLoadPointCloudCache( @@ -380,8 +381,14 @@ bool TryLoadPointCloudCache( const usdpointcloud::PointReadRequest& request, const std::string& parserVersion, bool& hit, - std::string& errorMessage) { + std::string& errorMessage, + cache::CacheDecision* decision) { hit = false; + const auto report = [decision](cache::CacheDecision value) { + if (decision) { + *decision = value; + } + }; if (!layer) { errorMessage = "cache lookup requires a writable layer"; return false; @@ -404,18 +411,25 @@ bool TryLoadPointCloudCache( const auto lookup = usdgeo::cache::Inspect(layout); if (lookup.status == usdgeo::cache::LookupStatus::Incomplete) { usdgeo::cache::Invalidate(cacheRoot, descriptor); + report(usdgeo::cache::CacheDecision::Invalidated); + return true; } if (!lookup.IsHit()) { + if (usdgeo::cache::HasSupersededIdentityEntry(layout)) { + report(usdgeo::cache::CacheDecision::IdentityChanged); + } return true; } const auto cachedLayer = pxr::SdfLayer::FindOrOpen(layout.rootLayer.string()); if (!cachedLayer) { usdgeo::cache::Invalidate(cacheRoot, descriptor); + report(usdgeo::cache::CacheDecision::Invalidated); return true; } if (!ValidateCachedPayloads(cachedLayer, layout)) { usdgeo::cache::Invalidate(cacheRoot, descriptor); + report(usdgeo::cache::CacheDecision::Invalidated); return true; } @@ -446,6 +460,7 @@ bool TryLoadPointCloudCache( : targetPayloadDirectory, layerBaseDirectory); hit = true; + report(usdgeo::cache::CacheDecision::Hit); return true; } diff --git a/libs/usd-pointcloud-authoring/tests/CMakeLists.txt b/libs/usd-pointcloud-authoring/tests/CMakeLists.txt index a96ac0e..fac618c 100644 --- a/libs/usd-pointcloud-authoring/tests/CMakeLists.txt +++ b/libs/usd-pointcloud-authoring/tests/CMakeLists.txt @@ -3,7 +3,9 @@ target_link_libraries(usdPointCloudAuthoring_tests PRIVATE usdpointcloud::authoring usdcopc::core) add_test(NAME usdPointCloudAuthoring_unit COMMAND usdPointCloudAuthoring_tests) +usdgeo_configure_openusd_test_runtime(usdPointCloudAuthoring_unit) add_executable(usdPointCloudCache_tests test_pointcloud_cache.cpp) target_link_libraries(usdPointCloudCache_tests PRIVATE usdpointcloud::authoring) add_test(NAME usdPointCloudCache_unit COMMAND usdPointCloudCache_tests) +usdgeo_configure_openusd_test_runtime(usdPointCloudCache_unit) diff --git a/openstrata.ci.yaml b/openstrata.ci.yaml index 877f5f3..c1baadf 100644 --- a/openstrata.ci.yaml +++ b/openstrata.ci.yaml @@ -393,3 +393,95 @@ cells: host_packages: apt: [libx11-dev, libxt-dev, libxext-dev, libgl1-mesa-dev] up_to: 5 + + # Workspace cells: the Tier 1 resolver contract gate. + # + # Every bundle cell above is a per-plugin build rooted at + # plugins/pointcloud-/CMakeLists.txt, a standalone project() that never + # declares USDGEO_BUILD_TESTS. The option is undefined there, so neither + # pointcloudCopc_tests nor the memory-backed resolver test double under + # tests/plugins/httpresolver is compiled by a bundle cell. Only the root + # CMakeLists.txt defaults USDGEO_BUILD_TESTS to ON, which is what a + # `kind: workspace` cell configures through `ost build` before running the + # CTest suite. These cells are therefore what makes Tier 1 a CI gate rather + # than a local-only one, and they need no external resolver repository. + + - name: workspace-pr-windows + lane: pull_request + runner: windows-hosted + kind: workspace + verify: test + runtime_artifact: sha256:c3ed40122756ea118166e1619efcaec463e7d2a42f6d978fe4e20b6b774c4b03 + runtime_remote: + uri: oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:07cb84412017ece911adaed1175a6373865d863bc40124081cfc20c52de7f0d9 + expected_oci_digest: sha256:07cb84412017ece911adaed1175a6373865d863bc40124081cfc20c52de7f0d9 + platform: cy2026 + profile: usd + + - name: workspace-pr-macos-arm64 + lane: pull_request + runner: macos-arm64-hosted + kind: workspace + verify: test + runtime_artifact: sha256:a9bb847ab5c7eb29d7425ff9acfb05b01c1751054d6a70628e48b06f8409a4a8 + runtime_remote: + uri: oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:dc0c980b868f91fe33e6f35a67a2bc5693b89f0243141c55ecb068ab225b0f3e + expected_oci_digest: sha256:dc0c980b868f91fe33e6f35a67a2bc5693b89f0243141c55ecb068ab225b0f3e + platform: cy2026 + profile: usd + host_python: "3.13" + + - name: workspace-pr-linux + lane: pull_request + runner: linux-hosted + kind: workspace + verify: test + runtime_artifact: sha256:03f7d4ef263abb50511d17237e7cbdbe1b83dee02cff8b8f901f0484c7a7898e + runtime_remote: + uri: oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:367a32bd1985ca9b07e8f7e64f95c1ded8dba16d68f17e0bffb09cec0b9dc3f6 + expected_oci_digest: sha256:367a32bd1985ca9b07e8f7e64f95c1ded8dba16d68f17e0bffb09cec0b9dc3f6 + platform: cy2026 + profile: usd + host_python: "3.13" + host_packages: + apt: [libx11-dev, libxt-dev, libxext-dev, libgl1-mesa-dev] + + - name: workspace-main-windows + lane: main + runner: windows-hosted + kind: workspace + verify: test + runtime_artifact: sha256:c3ed40122756ea118166e1619efcaec463e7d2a42f6d978fe4e20b6b774c4b03 + runtime_remote: + uri: oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:07cb84412017ece911adaed1175a6373865d863bc40124081cfc20c52de7f0d9 + expected_oci_digest: sha256:07cb84412017ece911adaed1175a6373865d863bc40124081cfc20c52de7f0d9 + platform: cy2026 + profile: usd + + - name: workspace-main-macos-arm64 + lane: main + runner: macos-arm64-hosted + kind: workspace + verify: test + runtime_artifact: sha256:a9bb847ab5c7eb29d7425ff9acfb05b01c1751054d6a70628e48b06f8409a4a8 + runtime_remote: + uri: oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:dc0c980b868f91fe33e6f35a67a2bc5693b89f0243141c55ecb068ab225b0f3e + expected_oci_digest: sha256:dc0c980b868f91fe33e6f35a67a2bc5693b89f0243141c55ecb068ab225b0f3e + platform: cy2026 + profile: usd + host_python: "3.13" + + - name: workspace-main-linux + lane: main + runner: linux-hosted + kind: workspace + verify: test + runtime_artifact: sha256:03f7d4ef263abb50511d17237e7cbdbe1b83dee02cff8b8f901f0484c7a7898e + runtime_remote: + uri: oci://ghcr.io/animu-sphere/openstrata-runtime-cy2026-usd@sha256:367a32bd1985ca9b07e8f7e64f95c1ded8dba16d68f17e0bffb09cec0b9dc3f6 + expected_oci_digest: sha256:367a32bd1985ca9b07e8f7e64f95c1ded8dba16d68f17e0bffb09cec0b9dc3f6 + platform: cy2026 + profile: usd + host_python: "3.13" + host_packages: + apt: [libx11-dev, libxt-dev, libxext-dev, libgl1-mesa-dev] diff --git a/openstrata.toml b/openstrata.toml index c26cfbc..e8f0160 100644 --- a/openstrata.toml +++ b/openstrata.toml @@ -1,6 +1,6 @@ [project] name = "usd-pointcloud-plugins" -version = "0.9.0" +version = "0.10.0" [requires] platform = "cy2026" diff --git a/plugins/pointcloud-copc/CMakeLists.txt b/plugins/pointcloud-copc/CMakeLists.txt index 47d47b5..0e3a356 100644 --- a/plugins/pointcloud-copc/CMakeLists.txt +++ b/plugins/pointcloud-copc/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.23) project(pointcloudCopc - VERSION 0.9.0 + VERSION 0.10.0 DESCRIPTION "OpenUSD FileFormat Plugin for COPC point clouds" LANGUAGES CXX) @@ -126,9 +126,9 @@ if(USDGEO_BUILD_TESTS) set_target_properties(pointcloudCopc_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/lib") add_test(NAME pointcloudCopc_integration COMMAND pointcloudCopc_tests) - set_tests_properties(pointcloudCopc_integration PROPERTIES - ENVIRONMENT_MODIFICATION - "PXR_AR_DEFAULT_RESOLVER=set:HttpResolver") + usdgeo_configure_openusd_test_runtime(pointcloudCopc_integration) + set_property(TEST pointcloudCopc_integration APPEND PROPERTY + ENVIRONMENT "PXR_AR_DEFAULT_RESOLVER=HttpResolver") endif() openstrata_configure_plugin( diff --git a/plugins/pointcloud-copc/README.md b/plugins/pointcloud-copc/README.md index d2db5f7..1bc914e 100644 --- a/plugins/pointcloud-copc/README.md +++ b/plugins/pointcloud-copc/README.md @@ -26,8 +26,25 @@ rather than LAS source order; bounds and classification filters are applied while decoded points pass through a bounded buffer. Remote COPC is supported only when the active resolver supplies an asset with efficient random-access reads, such as HTTP byte-range support. The plugin does not implement an HTTP -client, transport retries, or a network cache. Generated-USDC cache lookup is -limited to stable local filesystem identities; resolver-backed sources are -not reused when that identity cannot be established. The integration baseline -keeps this behavior covered with `USDGEO_CACHE_ROOT` configured. Remote tiled -reads require an absolute local `payloadDirectory`. +client, transport retries, or a network cache. Remote tiled reads require an +absolute local `payloadDirectory`. + +Generated-USDC cache lookup is enabled for a stable local filesystem identity +and for a `Stable` resolver identity. `Unstable` and `Unavailable` identity +fails closed: the read proceeds from the source and says why. Nothing here +publishes an entry - `usd-pointcloud-convert` does, and it accepts `.las` and +`.laz` local inputs only, so a COPC read has nothing to reuse outside a test +that commits an entry itself. + +Every cache decision is reported through four codes that project the stable +categories `usdgeo::cache` publishes, each message naming its exact category: + +| Code | Severity | Categories | +| --- | --- | --- | +| `COPC009` | warning | `resolver-identity-unavailable`, `resolver-identity-unstable`, `generated-cache-reuse-disabled` | +| `COPC010` | status | `resolver-identity-stable`, `generated-cache-hit` | +| `COPC011` | status | `resolver-identity-changed` | +| `COPC012` | warning | `generated-cache-invalidated` | + +Interoperability with an external resolver is recorded in the +[resolver read baseline](../../docs/reference/RESOLVER_BASELINE.md). diff --git a/plugins/pointcloud-copc/include/usdgeocopc/UsdGeoCopcDiagnostics.h b/plugins/pointcloud-copc/include/usdgeocopc/UsdGeoCopcDiagnostics.h index c988748..d10ba20 100644 --- a/plugins/pointcloud-copc/include/usdgeocopc/UsdGeoCopcDiagnostics.h +++ b/plugins/pointcloud-copc/include/usdgeocopc/UsdGeoCopcDiagnostics.h @@ -1,5 +1,7 @@ #pragma once +#include "usdgeo/cache/Cache.h" + #include namespace usdgeocopc::diagnostics { @@ -12,10 +14,60 @@ inline constexpr const char* UsdLayerCreateFailed = "COPC005"; inline constexpr const char* StageMetricsFailed = "COPC006"; inline constexpr const char* PointCloudAuthorFailed = "COPC007"; inline constexpr const char* FormatArgumentInvalid = "COPC008"; +// Generated-cache decision codes. Each one projects a group of the stable, +// transport-neutral categories `usdgeo::cache::CacheDecisionName` publishes; +// the emitted message always names the exact category it carries. +// +// COPC009 reuse disabled resolver-identity-unavailable +// resolver-identity-unstable +// generated-cache-reuse-disabled +// COPC010 reuse permitted resolver-identity-stable +// generated-cache-hit +// COPC011 regeneration resolver-identity-changed +// COPC012 entry removed generated-cache-invalidated inline constexpr const char* ResolverCacheReuseDisabled = "COPC009"; +inline constexpr const char* ResolverCacheReusePermitted = "COPC010"; +inline constexpr const char* ResolverIdentityChanged = "COPC011"; +inline constexpr const char* ResolverCacheInvalidated = "COPC012"; inline std::string Message(const char* code, const std::string& message) { return "[" + std::string(code) + "] " + message; } +// The projection itself, so what a test asserts is what OpenUSD is told. The +// message text is fixed by `usdgeo::cache` and the category name is an +// enumerated constant, so neither can carry a resolved identifier, a +// validation token, or any transport detail. +inline const char* DecisionCode(usdgeo::cache::CacheDecision decision) { + switch (decision) { + case usdgeo::cache::CacheDecision::IdentityStable: + case usdgeo::cache::CacheDecision::Hit: + return ResolverCacheReusePermitted; + case usdgeo::cache::CacheDecision::IdentityChanged: + return ResolverIdentityChanged; + case usdgeo::cache::CacheDecision::Invalidated: + return ResolverCacheInvalidated; + case usdgeo::cache::CacheDecision::IdentityUnavailable: + case usdgeo::cache::CacheDecision::IdentityUnstable: + case usdgeo::cache::CacheDecision::ReuseDisabled: + break; + } + return ResolverCacheReuseDisabled; +} + +// A decision that removed or refused something is a warning; one that reports +// what reuse did is a status. +inline bool DecisionIsWarning(usdgeo::cache::CacheDecision decision) { + const auto* code = DecisionCode(decision); + return code == ResolverCacheReuseDisabled || + code == ResolverCacheInvalidated; +} + +inline std::string DecisionMessage(usdgeo::cache::CacheDecision decision) { + return Message( + DecisionCode(decision), + std::string(usdgeo::cache::CacheDecisionMessage(decision)) + " (" + + usdgeo::cache::CacheDecisionName(decision) + ")"); +} + } // namespace usdgeocopc::diagnostics diff --git a/plugins/pointcloud-copc/openstrata.plugin.yaml b/plugins/pointcloud-copc/openstrata.plugin.yaml index b498046..7fb6faf 100644 --- a/plugins/pointcloud-copc/openstrata.plugin.yaml +++ b/plugins/pointcloud-copc/openstrata.plugin.yaml @@ -1,6 +1,6 @@ plugin: name: pointcloud-copc - version: 0.9.0 + version: 0.10.0 kind: usd-fileformat license: Apache-2.0 runtime: diff --git a/plugins/pointcloud-copc/src/UsdGeoCopcFileFormat.cpp b/plugins/pointcloud-copc/src/UsdGeoCopcFileFormat.cpp index 3d10b78..fbe8a07 100644 --- a/plugins/pointcloud-copc/src/UsdGeoCopcFileFormat.cpp +++ b/plugins/pointcloud-copc/src/UsdGeoCopcFileFormat.cpp @@ -72,6 +72,19 @@ const char* ReaderDiagnosticCode( return usdgeocopc::diagnostics::DecodeFailed; } +// Every generated-cache decision reaches OpenUSD through one of four plugin +// codes and always carries the stable category name, so a consumer can match +// on the category rather than on prose. The projection lives in the +// diagnostics header so a test asserts the same code and text OpenUSD is told. +void ReportCacheDecision(usdgeo::cache::CacheDecision decision) { + const auto message = usdgeocopc::diagnostics::DecisionMessage(decision); + if (usdgeocopc::diagnostics::DecisionIsWarning(decision)) { + TF_WARN("%s", message.c_str()); + } else { + TF_STATUS("%s", message.c_str()); + } +} + bool IsLocalFileSource(const std::string& path) { std::error_code error; return std::filesystem::is_regular_file(std::filesystem::path(path), error) && @@ -460,31 +473,43 @@ bool UsdGeoCopcFileFormat::Read(SdfLayer* layer, bool cacheHit = false; std::string cacheError; const auto loadCache = [&]() { + auto decision = usdgeo::cache::CacheDecision::ReuseDisabled; if (IsLocalFileSource(resolvedPath)) { - return usdgeo::TryLoadPointCloudCache( + const auto loaded = usdgeo::TryLoadPointCloudCache( layer, resolvedPath, reference, request, "copc-reader-1", - cacheHit, cacheError); + cacheHit, cacheError, &decision); + if (loaded && decision != + usdgeo::cache::CacheDecision::ReuseDisabled) { + ReportCacheDecision(decision); + } + return loaded; } if (resolverStability != usdgeo::cache::ResolverIdentityStability::Stable) { - const auto stabilityName = - usdgeo::cache::ResolverIdentityStabilityName( - resolverStability); - TF_WARN("[%s] Generated cache reuse disabled: the active " - "resolver did not provide a stable source validation " - "identity (%s).", - usdgeocopc::diagnostics::ResolverCacheReuseDisabled, - stabilityName); + // The identity category is the reason reuse is disabled, so it + // is what the diagnostic reports. + ReportCacheDecision( + usdgeo::cache::IdentityDecision(resolverStability)); return true; } const std::filesystem::path payloadDirectory(request.payloadDirectory); if (!payloadDirectory.empty() && payloadDirectory.is_relative()) { + // A remote source has no directory to resolve a relative + // payload path against, so reuse is refused even though the + // identity would permit it. + ReportCacheDecision( + usdgeo::cache::CacheDecision::ReuseDisabled); return true; } - return usdgeo::TryLoadPointCloudCache( + decision = usdgeo::cache::CacheDecision::IdentityStable; + const auto loaded = usdgeo::TryLoadPointCloudCache( layer, resolverIdentity, {}, reference, request, - "copc-reader-1", cacheHit, cacheError); + "copc-reader-1", cacheHit, cacheError, &decision); + if (loaded) { + ReportCacheDecision(decision); + } + return loaded; }; if (!loadCache()) { TF_RUNTIME_ERROR("%s", usdgeocopc::diagnostics::Message( diff --git a/plugins/pointcloud-copc/tests/test_pointcloud_copc.cpp b/plugins/pointcloud-copc/tests/test_pointcloud_copc.cpp index fc687ac..3aab50c 100644 --- a/plugins/pointcloud-copc/tests/test_pointcloud_copc.cpp +++ b/plugins/pointcloud-copc/tests/test_pointcloud_copc.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -183,6 +184,83 @@ void TestResolverCacheDiagnostic() { "[COPC009] Generated cache reuse disabled: the active resolver " "did not provide a stable source validation identity.", "resolver cache diagnostic format"); + + // The four decision codes are distinct and stable, and each emitted + // message ends with the shared category name so a consumer can match the + // category instead of the prose. + const std::vector codes{ + usdgeocopc::diagnostics::ResolverCacheReuseDisabled, + usdgeocopc::diagnostics::ResolverCacheReusePermitted, + usdgeocopc::diagnostics::ResolverIdentityChanged, + usdgeocopc::diagnostics::ResolverCacheInvalidated}; + for (std::size_t index = 0; index != codes.size(); ++index) { + for (std::size_t other = index + 1; other != codes.size(); ++other) { + Check(std::string(codes[index]) != codes[other], + "resolver decision codes must be distinct"); + } + } + Check(std::string(usdgeocopc::diagnostics::ResolverCacheReusePermitted) == + "COPC010" && + std::string(usdgeocopc::diagnostics::ResolverIdentityChanged) == + "COPC011" && + std::string(usdgeocopc::diagnostics::ResolverCacheInvalidated) == + "COPC012", + "resolver decision codes changed meaning"); + + // Assert the projection the plugin actually emits, category by category. + const std::vector> + projection{ + {usdgeo::cache::CacheDecision::IdentityUnavailable, + usdgeocopc::diagnostics::ResolverCacheReuseDisabled}, + {usdgeo::cache::CacheDecision::IdentityUnstable, + usdgeocopc::diagnostics::ResolverCacheReuseDisabled}, + {usdgeo::cache::CacheDecision::ReuseDisabled, + usdgeocopc::diagnostics::ResolverCacheReuseDisabled}, + {usdgeo::cache::CacheDecision::IdentityStable, + usdgeocopc::diagnostics::ResolverCacheReusePermitted}, + {usdgeo::cache::CacheDecision::Hit, + usdgeocopc::diagnostics::ResolverCacheReusePermitted}, + {usdgeo::cache::CacheDecision::IdentityChanged, + usdgeocopc::diagnostics::ResolverIdentityChanged}, + {usdgeo::cache::CacheDecision::Invalidated, + usdgeocopc::diagnostics::ResolverCacheInvalidated}}; + for (const auto& [decision, expectedCode] : projection) { + Check(std::string(usdgeocopc::diagnostics::DecisionCode(decision)) == + expectedCode, + "resolver decision projected onto the wrong code"); + const auto rendered = + usdgeocopc::diagnostics::DecisionMessage(decision); + Check(rendered.rfind(std::string("[") + expectedCode + "] ", 0) == 0, + "resolver decision message must lead with its code"); + Check(rendered.find(usdgeo::cache::CacheDecisionMessage(decision)) != + std::string::npos, + "resolver decision message must carry the shared text"); + // A consumer matches the category, so the category has to survive the + // projection. Building the expectation from the code path under test + // would assert nothing, so it is spelled out here. + Check(rendered.size() > 3 && + rendered.compare( + rendered.size() - + std::strlen( + usdgeo::cache::CacheDecisionName(decision)) - 2, + std::string::npos, + std::string("(") + + usdgeo::cache::CacheDecisionName(decision) + ")") == 0, + "resolver decision message must end with its category"); + for (const char* forbidden : {"http", "ETag", "etag", "Authorization"}) { + Check(rendered.find(forbidden) == std::string::npos, + "resolver decision message leaked a transport detail"); + } + } + Check(usdgeocopc::diagnostics::DecisionIsWarning( + usdgeo::cache::CacheDecision::ReuseDisabled) && + usdgeocopc::diagnostics::DecisionIsWarning( + usdgeo::cache::CacheDecision::Invalidated) && + !usdgeocopc::diagnostics::DecisionIsWarning( + usdgeo::cache::CacheDecision::Hit) && + !usdgeocopc::diagnostics::DecisionIsWarning( + usdgeo::cache::CacheDecision::IdentityChanged), + "resolver decision severity changed"); } void RegisterPlugin(const std::filesystem::path& plugInfo) { @@ -825,6 +903,57 @@ void TestResolverBackedRead() { Check(changedPositions.front() != firstPositions.front(), "resolver read reused cached output"); + // The regenerated entry lands beside the one it supersedes, under the same + // generation directory. That adjacency is what `resolver-identity-changed` + // reports; without it a changed revision is indistinguishable from a source + // that was never generated. + const auto changedAsset = pxr::ArGetResolver().OpenAsset( + pxr::ArResolvedPath("http://memory.copc")); + Check(static_cast(changedAsset), "reopen changed resolver asset"); + usdgeo::cache::SourceIdentity changedIdentity; + usdgeo::cache::ResolverIdentityStability changedStability; + Check(usdgeo::TryBuildResolverSourceIdentity( + pxr::ArGetResolver(), "http://memory.copc", + pxr::ArResolvedPath("http://memory.copc"), *changedAsset, + changedIdentity, changedStability, identityError), + "changed resolver identity"); + Check(changedStability == + usdgeo::cache::ResolverIdentityStability::Stable); + Check(changedIdentity.validationToken != initialValidationToken); + // Rebuild the georeference from the changed source rather than reusing the + // original: asserting adjacency against a reference this test already holds + // would hold by construction. This fixture writes a fixed header bounding + // box, so the rebuilt reference happens to match, and the case where a + // revision *does* move the georeference is covered directly in + // usdGeoCache's TestSupersededIdentityEntry. + auto changedSource = + std::make_shared( + changedAsset, "http://memory.copc"); + usdcopc::CopcReader changedReader(changedSource); + usdcopc::CopcHeader changedHeader; + std::vector changedDiagnostics; + Check(changedReader.ReadMetadata(changedHeader, changedDiagnostics), + "read changed fixture metadata"); + usdpointcloud::PointChunk changedMetadataChunk; + usdgeo::GeoReference changedReference; + usdgeo::SpatialBounds changedBounds; + Check(usdlas::BuildPointCloudMetadata( + changedHeader.las, changedMetadataChunk, changedReference, + changedBounds, cacheErrorMessage), + "build changed fixture metadata"); + usdgeo::cache::Layout changedLayout; + Check(usdgeo::TryBuildPointCloudCacheLayout( + cacheRoot, changedIdentity, changedReference, cacheRequest, + "copc-reader-1", changedLayout, cacheErrorMessage), + "build changed resolver cache layout"); + Check(changedLayout.entryDirectory != cacheLayout.entryDirectory, + "changed validation identity reused the superseded entry"); + Check(changedLayout.entryDirectory.parent_path() == + cacheLayout.entryDirectory.parent_path(), + "changed validation identity left the generation directory"); + Check(usdgeo::cache::HasSupersededIdentityEntry(changedLayout), + "changed validation identity did not observe the superseded entry"); + std::error_code error; std::filesystem::remove_all(cacheRoot, error); std::filesystem::remove(lazPath, error); diff --git a/plugins/pointcloud-las/CMakeLists.txt b/plugins/pointcloud-las/CMakeLists.txt index 3c3da9e..bcd5b4d 100644 --- a/plugins/pointcloud-las/CMakeLists.txt +++ b/plugins/pointcloud-las/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.23) project(pointcloudLas - VERSION 0.9.0 + VERSION 0.10.0 DESCRIPTION "OpenUSD FileFormat Plugin for LAS point clouds" LANGUAGES CXX) @@ -91,6 +91,7 @@ if(USDGEO_BUILD_TESTS) set_target_properties(pointcloudLas_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/lib") add_test(NAME pointcloudLas_integration COMMAND pointcloudLas_tests) + usdgeo_configure_openusd_test_runtime(pointcloudLas_integration) endif() openstrata_configure_plugin( diff --git a/plugins/pointcloud-las/openstrata.plugin.yaml b/plugins/pointcloud-las/openstrata.plugin.yaml index e673360..fe34863 100644 --- a/plugins/pointcloud-las/openstrata.plugin.yaml +++ b/plugins/pointcloud-las/openstrata.plugin.yaml @@ -1,6 +1,6 @@ plugin: name: pointcloud-las - version: 0.9.0 + version: 0.10.0 kind: usd-fileformat license: Apache-2.0 runtime: diff --git a/plugins/pointcloud-laz/CMakeLists.txt b/plugins/pointcloud-laz/CMakeLists.txt index 48a8c56..6d80150 100644 --- a/plugins/pointcloud-laz/CMakeLists.txt +++ b/plugins/pointcloud-laz/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.23) project(pointcloudLaz - VERSION 0.9.0 + VERSION 0.10.0 DESCRIPTION "OpenUSD FileFormat Plugin for LAZ point clouds" LANGUAGES CXX) @@ -99,6 +99,7 @@ if(USDGEO_BUILD_TESTS) set_target_properties(pointcloudLaz_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/lib") add_test(NAME pointcloudLaz_integration COMMAND pointcloudLaz_tests) + usdgeo_configure_openusd_test_runtime(pointcloudLaz_integration) endif() openstrata_configure_plugin( diff --git a/plugins/pointcloud-laz/openstrata.plugin.yaml b/plugins/pointcloud-laz/openstrata.plugin.yaml index 0752a87..097002e 100644 --- a/plugins/pointcloud-laz/openstrata.plugin.yaml +++ b/plugins/pointcloud-laz/openstrata.plugin.yaml @@ -1,6 +1,6 @@ plugin: name: pointcloud-laz - version: 0.9.0 + version: 0.10.0 kind: usd-fileformat license: Apache-2.0 runtime: diff --git a/plugins/pointcloud-ply/CMakeLists.txt b/plugins/pointcloud-ply/CMakeLists.txt index 26f25d3..e1d539b 100644 --- a/plugins/pointcloud-ply/CMakeLists.txt +++ b/plugins/pointcloud-ply/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.23) project(pointcloudPly - VERSION 0.9.0 + VERSION 0.10.0 DESCRIPTION "OpenUSD FileFormat Plugin for PLY point clouds" LANGUAGES CXX) @@ -84,6 +84,7 @@ if(USDGEO_BUILD_TESTS) set_target_properties(pointcloudPly_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/lib") add_test(NAME pointcloudPly_integration COMMAND pointcloudPly_tests) + usdgeo_configure_openusd_test_runtime(pointcloudPly_integration) endif() openstrata_configure_plugin( diff --git a/plugins/pointcloud-ply/openstrata.plugin.yaml b/plugins/pointcloud-ply/openstrata.plugin.yaml index e3c0402..6033428 100644 --- a/plugins/pointcloud-ply/openstrata.plugin.yaml +++ b/plugins/pointcloud-ply/openstrata.plugin.yaml @@ -1,6 +1,6 @@ plugin: name: pointcloud-ply - version: 0.9.0 + version: 0.10.0 kind: usd-fileformat license: Apache-2.0 runtime: diff --git a/plugins/pointcloud-ply/tests/test_pointcloud_ply.cpp b/plugins/pointcloud-ply/tests/test_pointcloud_ply.cpp index cf33f9a..3d732f6 100644 --- a/plugins/pointcloud-ply/tests/test_pointcloud_ply.cpp +++ b/plugins/pointcloud-ply/tests/test_pointcloud_ply.cpp @@ -87,7 +87,7 @@ void TestFileFormatIntegration() { 4096); const auto tiledPayloadDirectory = - std::filesystem::temp_directory_path() / "usd_geo_ply_tiled_payloads"; + source.parent_path() / "usd_geo_ply_tiled_payloads"; std::filesystem::remove_all(tiledPayloadDirectory); const pxr::SdfLayer::FileFormatArguments tiledArguments = { {"epsg", "4978"}, diff --git a/tests/plugins/httpresolver/CMakeLists.txt b/tests/plugins/httpresolver/CMakeLists.txt index e987427..a3c9f07 100644 --- a/tests/plugins/httpresolver/CMakeLists.txt +++ b/tests/plugins/httpresolver/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.23) project(httpResolver - VERSION 0.9.0 + VERSION 0.10.0 DESCRIPTION "OpenUSD HTTP resolver test double for point-cloud integration" LANGUAGES CXX) diff --git a/tools/tier2_fixture_server.py b/tools/tier2_fixture_server.py new file mode 100644 index 0000000..88e4d46 --- /dev/null +++ b/tools/tier2_fixture_server.py @@ -0,0 +1,167 @@ +"""Static range-serving HTTP origin for the Tier 2 resolver integration. + +This is a measurement fixture, not a transport. It serves one file over +loopback, honours a single `Range: bytes=` request, and writes a request log so +the bytes an external resolver actually fetched can be divided by the source +size. Nothing in the product depends on it, and nothing here knows what a COPC +is. + + python tools/tier2_fixture_server.py --file --route /data.copc \ + --log [--port 0] + +The chosen port is printed as `port=` on the first line of stdout, so a +driver can bind port 0 and read back what the kernel assigned. +""" + +from __future__ import annotations + +import argparse +import http.server +import json +import os +import re +import threading +import time + +RANGE = re.compile(r"^bytes=(\d*)-(\d*)$") + + +def _write_log() -> None: + if not Handler.log_path: + return + payload = {"sourceBytes": len(Handler.payload), "requests": Handler.log} + temporary = Handler.log_path + ".tmp" + with open(temporary, "w", encoding="utf-8") as output: + json.dump(payload, output, indent=2) + os.replace(temporary, Handler.log_path) + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + # Written by the server owner before serving. + payload: bytes = b"" + route: str = "/" + validator: str = "" + log: list = [] + log_path: str = "" + lock = threading.Lock() + + def log_message(self, *args): # noqa: D102 - quiet; the JSON log is the record + return + + def _record(self, method: str, status: int, sent: int) -> None: + # Flushed on every request rather than at shutdown: the driver stops + # this process by killing it, and a log that only exists after a clean + # exit is a log that is not there when the measurement needs it. + with Handler.lock: + Handler.log.append({ + "method": method, + "path": self.path, + "range": self.headers.get("Range", ""), + "status": status, + "bytesSent": sent, + "monotonic": time.monotonic(), + }) + _write_log() + + def _reject(self, method: str, status: int) -> None: + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + self._record(method, status, 0) + + def _common_headers(self, length: int) -> None: + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Accept-Ranges", "bytes") + self.send_header("ETag", Handler.validator) + self.send_header("Content-Length", str(length)) + + def do_HEAD(self) -> None: # noqa: N802 - http.server naming + if self.path != Handler.route: + self._reject("HEAD", 404) + return + self.send_response(200) + self._common_headers(len(Handler.payload)) + self.end_headers() + self._record("HEAD", 200, 0) + + def do_GET(self) -> None: # noqa: N802 - http.server naming + if self.path != Handler.route: + self._reject("GET", 404) + return + total = len(Handler.payload) + header = self.headers.get("Range") + if not header: + self.send_response(200) + self._common_headers(total) + self.end_headers() + self.wfile.write(Handler.payload) + self._record("GET", 200, total) + return + + match = RANGE.match(header.strip()) + if not match: + self._reject("GET", 416) + return + first, last = match.group(1), match.group(2) + if first == "": + if last == "": + self._reject("GET", 416) + return + start, end = max(0, total - int(last)), total - 1 + else: + start = int(first) + end = total - 1 if last == "" else min(int(last), total - 1) + if start > end or start >= total: + self.send_response(416) + self.send_header("Content-Range", f"bytes */{total}") + self.send_header("Content-Length", "0") + self.end_headers() + self._record("GET", 416, 0) + return + + body = Handler.payload[start:end + 1] + self.send_response(206) + self.send_header("Content-Range", f"bytes {start}-{end}/{total}") + self._common_headers(len(body)) + self.end_headers() + self.wfile.write(body) + self._record("GET", 206, len(body)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--file", required=True) + parser.add_argument("--route", default="/fixture.copc") + parser.add_argument("--log", required=True) + parser.add_argument("--port", type=int, default=0) + parser.add_argument("--validator", default="") + arguments = parser.parse_args() + + with open(arguments.file, "rb") as source: + Handler.payload = source.read() + Handler.route = arguments.route + Handler.validator = arguments.validator or '"{:x}-{:x}"'.format( + len(Handler.payload), os.stat(arguments.file).st_mtime_ns) + Handler.log = [] + Handler.log_path = arguments.log + _write_log() + + server = http.server.ThreadingHTTPServer(("127.0.0.1", arguments.port), + Handler) + print(f"port={server.server_address[1]}", flush=True) + print(f"size={len(Handler.payload)}", flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + with Handler.lock: + _write_log() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tier2_resolver_integration.py b/tools/tier2_resolver_integration.py new file mode 100644 index 0000000..5e51c57 --- /dev/null +++ b/tools/tier2_resolver_integration.py @@ -0,0 +1,296 @@ +"""Tier 2 resolver interoperability harness. + +Composes an external OpenUSD resolver, a local range-serving HTTP origin, and a +COPC fixture, then records what a remote read actually costs. It is a +measurement driver: it links nothing, and this repository keeps no build edge to +any resolver implementation. + +Run it inside an activated OpenStrata runtime, e.g. + + python tools/tier2_resolver_integration.py \ + --fixture build/real-data-source/autzen-classified.copc.laz \ + --resolver-resources /plugin/resources/httpResolver \ + --copc-resources plugins/pointcloud-copc/plugin/resources/pointcloud-copc \ + --output build/tier2/record.json + +Each scenario runs in a fresh interpreter against a fresh origin, so no +in-process resolver state and no request log is carried between rows and +`bytesFetched` is that scenario's cost alone. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +ROUTE = "/fixture.copc" + + +# -------------------------------------------------------------------------- +# Scenario bodies. These run in the child interpreter, one per invocation. +# -------------------------------------------------------------------------- + +def _summarize_stage(layer) -> dict: + from pxr import Sdf, Usd, UsdGeom + + stage = Usd.Stage.Open(layer) + points = UsdGeom.Points.Get(stage, Sdf.Path("/PointCloud")) + result = {"prims": sorted(prim.name for prim in layer.rootPrims)} + if not points: + return result + positions = points.GetPointsAttr().Get() + result["pointCount"] = 0 if positions is None else len(positions) + digest = hashlib.sha256() + if positions is not None: + for position in positions: + digest.update( + ("%.6f,%.6f,%.6f;" % (position[0], position[1], position[2])) + .encode("ascii")) + result["pointDigest"] = digest.hexdigest() + extent = points.GetExtentAttr().Get() + if extent is not None: + result["extent"] = [[float(value) for value in corner] + for corner in extent] + return result + + +def run_scenario(name: str, target: str) -> dict: + from pxr import Ar, Sdf + + record = {"scenario": name, "target": target} + resolver = Ar.GetResolver() + if target.startswith("http"): + resolved = resolver.Resolve(target) + info = resolver.GetAssetInfo(target, resolved) + identifier = resolved.GetPathString() + # The token is opaque to this repository and is never recorded. Only + # whether one exists, and a digest that lets a later revision be seen + # to differ without the value itself reaching the record. + record["hasValidationToken"] = bool(info.version) + record["validationTokenDigest"] = ( + hashlib.sha256(info.version.encode("utf-8")).hexdigest()[:16] + if info.version else "") + record["identityClass"] = ( + "stable" if identifier and info.version + else "unstable" if identifier else "unavailable") + + start = time.monotonic() + if name == "metadata": + layer = Sdf.Layer.OpenAsAnonymous(target, metadataOnly=True) + record["opened"] = bool(layer) + if layer: + record["prims"] = sorted(prim.name for prim in layer.rootPrims) + else: + layer = Sdf.Layer.FindOrOpen(target) + record["opened"] = bool(layer) + if layer: + record.update(_summarize_stage(layer)) + record["elapsedSeconds"] = round(time.monotonic() - start, 4) + return record + + +# -------------------------------------------------------------------------- +# Driver. +# -------------------------------------------------------------------------- + +def reserve_port() -> int: + """One port for the whole run. + + Every revision has to be served at the *same* URL, or the run is not three + revisions of one identifier - it is three unrelated assets, and the property + being demonstrated (equal identifier, different validator) is not being + demonstrated at all. Origins run one at a time, so a single reserved port is + enough. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +class Origin: + """The loopback fixture origin, plus the request log it writes.""" + + def __init__(self, server: Path, fixture: Path, log: Path, validator: str, + port: int): + self.log = log + self.process = subprocess.Popen( + [sys.executable, str(server), "--file", str(fixture), + "--route", ROUTE, "--log", str(log), "--port", str(port), + "--validator", validator], + stdout=subprocess.PIPE, text=True) + try: + self.port = int(self._line().split("=", 1)[1]) + self.size = int(self._line().split("=", 1)[1]) + except BaseException: + # The handshake can fail after the process exists; leaving it + # running would hold the reserved port for the rest of the run. + self.close() + raise + + def _line(self) -> str: + line = self.process.stdout.readline() + if not line: + raise RuntimeError("fixture origin did not start") + return line.strip() + + @property + def url(self) -> str: + return "http://127.0.0.1:%d%s" % (self.port, ROUTE) + + def stats(self) -> dict: + with open(self.log, encoding="utf-8") as handle: + data = json.load(handle) + requests = data["requests"] + fetched = sum(entry["bytesSent"] for entry in requests) + source = data["sourceBytes"] + return {"requests": len(requests), + "rangeRequests": sum(1 for entry in requests + if entry["range"]), + "bytesFetched": fetched, + "sourceBytes": source, + "selectivity": round(fetched / source, 6) if source else 0.0} + + def close(self) -> None: + self.process.kill() + self.process.wait() + + +# The four generated-cache decision codes the COPC plugin emits. A scenario +# records which ones OpenUSD reported, because "reuse was disabled" is a claim +# about a diagnostic, not about a return value. +DECISION_CODES = ("COPC009", "COPC010", "COPC011", "COPC012") + + +def child(script: Path, environment: dict, name: str, target: str, + cache_root=None) -> dict: + child_environment = dict(environment) + if cache_root is not None: + cache_root.mkdir(parents=True, exist_ok=True) + child_environment["USDGEO_CACHE_ROOT"] = str(cache_root) + else: + child_environment.pop("USDGEO_CACHE_ROOT", None) + completed = subprocess.run( + [sys.executable, str(script), "--run-scenario", name, + "--target", target], + capture_output=True, text=True, env=child_environment) + if completed.returncode != 0: + raise RuntimeError("scenario %s failed:\n%s\n%s" + % (name, completed.stdout, completed.stderr)) + record = json.loads(completed.stdout.strip().splitlines()[-1]) + record["cacheRootConfigured"] = cache_root is not None + record["decisionCodes"] = [code for code in DECISION_CODES + if code in completed.stderr] + return record + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-scenario") + parser.add_argument("--target") + parser.add_argument("--fixture") + parser.add_argument("--resolver-resources") + parser.add_argument("--copc-resources") + parser.add_argument("--output") + arguments = parser.parse_args() + + if arguments.run_scenario: + print(json.dumps(run_scenario(arguments.run_scenario, + arguments.target))) + return 0 + + for required in ("fixture", "resolver_resources", "copc_resources", + "output"): + if not getattr(arguments, required): + parser.error("--%s is required" % required.replace("_", "-")) + + script = Path(__file__).resolve() + server = script.parent / "tier2_fixture_server.py" + fixture = Path(arguments.fixture).resolve() + output = Path(arguments.output).resolve() + output.parent.mkdir(parents=True, exist_ok=True) + + environment = dict(os.environ) + environment["PXR_PLUGINPATH_NAME"] = os.pathsep.join( + entry for entry in + [str(Path(arguments.copc_resources).resolve()), + str(Path(arguments.resolver_resources).resolve()), + environment.get("PXR_PLUGINPATH_NAME", "")] if entry) + + with open(fixture, "rb") as handle: + fixture_bytes = handle.read() + record = { + "fixture": { + "name": fixture.name, + "sizeBytes": len(fixture_bytes), + "sha256": hashlib.sha256(fixture_bytes).hexdigest(), + }, + "scenarios": [], + } + + workspace = Path(tempfile.mkdtemp(prefix="tier2-resolver-")) + try: + local = workspace / "local.copc" + shutil.copyfile(fixture, local) + + # Local baseline: the same bytes through the same FileFormat with no + # resolver in the path. Its authored output is the oracle the remote + # rows are compared against. + record["scenarios"].append( + child(script, environment, "full-local", str(local))) + + # Three revisions of one identifier, served at one reserved port so the + # identifier really is one identifier. A and B differ only in the + # validator, so a changed validation identity is visible without the + # bytes changing - identifier equality is not content equality. W serves + # a weak validator, which a resolver must not publish as a stable + # identity, and is how the conservative fallback is exercised against a + # real resolver rather than a test double. + port = reserve_port() + # One cache root for the whole run: separate roots per scenario would + # make every lookup a first lookup and hide any relationship between + # revisions. + cache_root = workspace / "cache" + revisions = (("A", '"revision-a"'), + ("B", '"revision-b"'), + ("W", 'W/"revision-w"')) + identifiers = set() + for revision, validator in revisions: + for name in ("metadata", "full"): + log = workspace / ("origin-%s-%s.json" % (revision, name)) + origin = Origin(server, fixture, log, validator, port) + try: + entry = child(script, environment, name, origin.url, + cache_root) + entry["revision"] = revision + entry["validatorStrength"] = ( + "weak" if validator.startswith("W/") else "strong") + entry["origin"] = origin.stats() + identifiers.add(origin.url) + record["scenarios"].append(entry) + finally: + origin.close() + if len(identifiers) != 1: + raise RuntimeError( + "revisions were served at %d identifiers, not one: %s" + % (len(identifiers), sorted(identifiers))) + record["identifier"] = identifiers.pop() + finally: + shutil.rmtree(workspace, ignore_errors=True) + + with open(output, "w", encoding="utf-8") as handle: + json.dump(record, handle, indent=2) + print(json.dumps(record, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/usd-pointcloud-convert/CMakeLists.txt b/tools/usd-pointcloud-convert/CMakeLists.txt index 0c737bb..65dcb61 100644 --- a/tools/usd-pointcloud-convert/CMakeLists.txt +++ b/tools/usd-pointcloud-convert/CMakeLists.txt @@ -35,10 +35,12 @@ install(TARGETS usd-pointcloud-convert if(USDGEO_BUILD_TESTS) add_test(NAME usdPointCloudConvert_help COMMAND usd-pointcloud-convert --help) + usdgeo_configure_openusd_test_runtime(usdPointCloudConvert_help) add_test(NAME usdPointCloudConvert_las COMMAND ${CMAKE_COMMAND} -Dconverter=$ -Dfixture=${CMAKE_SOURCE_DIR}/plugins/pointcloud-las/tests/fixtures/conformance.las -Dtest_root=${CMAKE_CURRENT_BINARY_DIR}/converter-conformance -P ${CMAKE_CURRENT_SOURCE_DIR}/test_conversion.cmake) + usdgeo_configure_openusd_test_runtime(usdPointCloudConvert_las) endif() diff --git a/tools/usd-pointcloud-convert/main.cpp b/tools/usd-pointcloud-convert/main.cpp index 98885e1..e6f8e42 100644 --- a/tools/usd-pointcloud-convert/main.cpp +++ b/tools/usd-pointcloud-convert/main.cpp @@ -341,7 +341,10 @@ bool BuildCacheDescriptor( errorMessage = "adaptive cache descriptor requires a tile plan key"; return false; } - descriptor.tileAndLod.emplace_back("tile.plan", tilePlanKey); + // The planner's identity and version are caller intent; the plan it + // produced is read out of the source, so it belongs in the + // source-derived half of the descriptor. + descriptor.sourceDerived.emplace_back("tile.plan", tilePlanKey); } descriptor.downsampling = {{"algorithm", "fixed-stride"}, {"version", "1"}}; diff --git a/tools/usd-pointcloud-convert/test_conversion.cmake b/tools/usd-pointcloud-convert/test_conversion.cmake index 1ee467d..130efd3 100644 --- a/tools/usd-pointcloud-convert/test_conversion.cmake +++ b/tools/usd-pointcloud-convert/test_conversion.cmake @@ -196,7 +196,8 @@ if(NOT cache_first_result EQUAL 0 OR NOT cache_first_error STREQUAL "" OR message(FATAL_ERROR "cached first conversion failed: ${cache_first_output}${cache_first_error}") endif() -file(GLOB cache_entries RELATIVE "${cache_root}" "${cache_root}/*") +# Cache entries are two levels deep: /. +file(GLOB cache_entries RELATIVE "${cache_root}" "${cache_root}/*/*") list(LENGTH cache_entries cache_entry_count) if(NOT cache_entry_count EQUAL 1 OR NOT EXISTS "${cache_root}/${cache_entries}/root.usdc" OR @@ -229,6 +230,30 @@ if(NOT cache_second_result EQUAL 0 OR NOT cache_second_error STREQUAL "" OR "${cache_second_output}${cache_second_error}") endif() +# No credential, signed URL, resolved identifier, or validation token is ever +# persisted into a manifest or a cache entry. The local path is the strongest +# identifier this fixture can produce and the fnv1a64 token is its validation +# material; neither may appear in any published artifact. +foreach(secret_manifest + "${cache_root}/${cache_entries}/cache.manifest" + "${cache_root}/${cache_entries}/payloads/tiles.manifest" + "${cache_second_root}/PointCloud.usda.manifest" + "${cache_second_root}/payloads/tiles.manifest") + file(READ "${secret_manifest}" secret_manifest_content) + string(FIND "${secret_manifest_content}" "fnv1a64:" secret_token_position) + get_filename_component(fixture_directory "${fixture}" DIRECTORY) + string(FIND "${secret_manifest_content}" "${fixture_directory}" + secret_path_position) + string(FIND "${secret_manifest_content}" "${cache_root}" + secret_cache_position) + if(NOT secret_token_position EQUAL -1 OR + NOT secret_path_position EQUAL -1 OR + NOT secret_cache_position EQUAL -1) + message(FATAL_ERROR + "manifest persisted source identity material: ${secret_manifest}") + endif() +endforeach() + set(cache_rebuild_root "${test_root}-cache-rebuild") set(cache_rebuild_output "${cache_rebuild_root}/PointCloud.usda") file(REMOVE_RECURSE "${cache_rebuild_root}")